From 5709e1a864c0e7b17c60e0f149084a2fede0c17c Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Fri, 3 Jan 2025 14:21:54 +0100 Subject: [PATCH 01/18] Add a SourceKit plugin to handle code completion requests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This adds a sourcekitd plugin that drives the code completion requests. It also includes a `CompletionScoring` module that’s used to rank code completion results based on their contextual match, allowing us to show more relevant code completion results at the top. --- Package.swift | 163 ++ .../Semantics/CompletionScore.swift | 75 + .../Semantics/Components/Availability.swift | 75 + .../Semantics/Components/CompletionKind.swift | 92 + .../Semantics/Components/Flair.swift | 139 ++ .../Components/ModuleProximity.swift | 71 + .../Semantics/Components/Popularity.swift | 125 + .../Components/PopularityIndex.swift | 221 ++ .../Components/PopularityTable.swift | 192 ++ .../Semantics/Components/ScopeProximity.swift | 81 + .../Components/StructuralProximity.swift | 58 + .../SynchronicityCompatibility.swift | 62 + .../Components/TypeCompatibility.swift | 77 + .../Semantics/SemanticClassification.swift | 362 +++ .../Text/CandidateBatch.swift | 584 +++++ .../Text/InfluencingIdentifiers.swift | 169 ++ .../Text/MatchCollator.Match.swift | 48 + .../Text/MatchCollator.Selection.swift | 22 + .../Text/MatchCollator.swift | 546 +++++ Sources/CompletionScoring/Text/Pattern.swift | 1276 ++++++++++ .../Text/RejectionFilter.swift | 135 ++ .../Text/ScoredMatchSelector.swift | 119 + Sources/CompletionScoring/Text/UTF8Byte.swift | 65 + .../Utilities/SelectTopK.swift | 134 ++ .../Serialization/BinaryCodable.swift | 66 + .../Serialization/BinaryDecoder.swift | 92 + .../Serialization/BinaryEncoder.swift | 59 + .../Conformances/Array+BinaryCodable.swift | 30 + .../Dictionary+BinaryCodable.swift | 36 + .../OptionSet+BinaryCodable.swift | 26 + .../Conformances/Optional+BinaryCodable.swift | 29 + .../Conformances/Scalars+BinaryCodable.swift | 103 + .../Conformances/String+BinaryCodable.swift | 27 + .../Utilities/SwiftExtensions.swift | 476 ++++ .../Utilities/UnsafeArray.swift | 91 + .../Utilities/UnsafeStackAllocator.swift | 245 ++ Sources/CompletionScoringForPlugin | 1 + .../CommonFunctionTerms.json | 994 ++++++++ .../RandomSeed.plist | 1030 ++++++++ .../RepeatableRandomNumberGenerator.swift | 116 + .../SymbolGenerator.swift | 201 ++ .../TestExtensions.swift | 237 ++ .../Timings.swift | 117 + .../WeightedChoices.swift | 34 + .../include/CodeCompletionSwiftInterop.h | 408 ++++ Sources/Csourcekitd/include/module.modulemap | 3 +- Sources/Csourcekitd/include/plugin.h | 473 ++++ .../include/sourcekitd_functions.h | 47 +- .../CustomLogStringConvertible.swift | 10 +- Sources/SKLoggingForPlugin | 1 + Sources/SKTestSupport/Assertions.swift | 18 + .../SKTestSupport/MultiFileTestProject.swift | 59 +- .../TestSourceKitLSPClient.swift | 8 +- Sources/SKTestSupport/Utils.swift | 30 +- Sources/SKUtilities/LineTable.swift | 46 + Sources/SKUtilitiesForPlugin | 1 + .../DynamicallyLoadedSourceKitD.swift | 124 +- Sources/SourceKitD/SKDResponseArray.swift | 10 + .../SourceKitD/SKDResponseDictionary.swift | 13 + Sources/SourceKitD/SourceKitD.swift | 39 +- Sources/SourceKitD/SourceKitDRegistry.swift | 37 +- Sources/SourceKitD/dlopen.swift | 4 + Sources/SourceKitD/sourcekitd_functions.swift | 193 ++ Sources/SourceKitD/sourcekitd_uids.swift | 87 + Sources/SourceKitD/sourcekitd_uids.swift.gyb | 42 +- Sources/SourceKitDForPlugin | 1 + Sources/SwiftExtensionsForPlugin | 1 + .../ClientPlugin.swift | 32 + .../ASTCompletion/ASTCompletionItem.swift | 647 ++++++ .../ASTCompletion/CompletionSession.swift | 283 +++ .../Completion+Diagnostic.swift | 32 + .../Completion+Identifier.swift | 33 + .../CodeCompletion/Completion+ItemKind.swift | 48 + .../Completion+PriorityBucket.swift | 69 + .../Completion+SemanticContext.swift | 49 + .../Completion+TypeRelation.swift | 34 + .../CodeCompletion/CompletionContext.swift | 68 + .../CodeCompletion/CompletionItem.swift | 58 + .../CodeCompletion/CompletionOptions.swift | 56 + .../CodeCompletion/CompletionSorting.swift | 84 + .../CodeCompletion/Connection.swift | 330 +++ .../CodeCompletion/Location.swift | 28 + .../CodeCompletion/Position.swift | 32 + .../CodeCompletion/TextEdit.swift | 25 + .../CodeCompletion/WithArrayOfCStrings.swift | 50 + .../CompletionProvider.swift | 510 ++++ .../CompletionResultsArrayBuilder.swift | 116 + Sources/SwiftSourceKitPlugin/Plugin.swift | 200 ++ .../SKDRequestArrayReader.swift | 69 + .../SKDRequestDictionaryReader.swift | 91 + .../SwiftSourceKitPlugin/SKDResponse.swift | 70 + .../SKDResponseArrayBuilder.swift | 70 + .../SKDResponseDictionaryBuilder.swift | 81 + .../SKDResponseValue.swift | 32 + .../SourceKitDWrappers.swift | 34 + .../SourceKitPluginError.swift | 32 + .../CompletionResultsArray.swift | 232 ++ ...llyLoadedSourceKitD+relativeToPlugin.swift | 58 + .../CandidateBatchPerfTests.swift | 52 + .../ScoringPerfTests.Completion.swift | 23 + .../ScoringPerfTests.Corpus.swift | 65 + .../ScoringPerfTests.Module.swift | 117 + .../ScoringPerfTests.Provider.swift | 72 + .../ScoringPerfTests.swift | 213 ++ ...deCompletionFoundationPerfTests-Info.plist | 22 + .../BinaryCodingTests.swift | 168 ++ .../CandidateBatchTests.swift | 1478 ++++++++++++ .../CompletionScoreTests.swift | 277 +++ .../InfluencingIdentifiersTests.swift | 91 + .../CompletionScoringTests/PatternTests.swift | 1301 +++++++++++ .../RejectionFilterTests.swift | 118 + .../SemanticScoredText.swift | 92 + .../StackAllocateTests.swift | 106 + .../CodeCompletionFoundationTests-Info.plist | 22 + .../SwiftExtensionsTests.swift | 161 ++ .../TestScoringWorkload.swift | 380 +++ .../CompletionScoringTests/TestTimings.swift | 71 + Tests/CompletionScoringTests/TopKTests.swift | 76 + .../UTF8ByteTests.swift | 60 + .../XCTestCaseScoringAdditions.swift | 138 ++ .../SourceKitDRegistryTests.swift | 5 +- .../PluginPaths.swift | 107 + .../PluginSwiftPMTestProject.swift | 114 + .../SwiftSourceKitPluginTests.swift | 2061 +++++++++++++++++ Utilities/build-script-helper.py | 14 +- 125 files changed, 21424 insertions(+), 89 deletions(-) create mode 100644 Sources/CompletionScoring/Semantics/CompletionScore.swift create mode 100644 Sources/CompletionScoring/Semantics/Components/Availability.swift create mode 100644 Sources/CompletionScoring/Semantics/Components/CompletionKind.swift create mode 100644 Sources/CompletionScoring/Semantics/Components/Flair.swift create mode 100644 Sources/CompletionScoring/Semantics/Components/ModuleProximity.swift create mode 100644 Sources/CompletionScoring/Semantics/Components/Popularity.swift create mode 100644 Sources/CompletionScoring/Semantics/Components/PopularityIndex.swift create mode 100644 Sources/CompletionScoring/Semantics/Components/PopularityTable.swift create mode 100644 Sources/CompletionScoring/Semantics/Components/ScopeProximity.swift create mode 100644 Sources/CompletionScoring/Semantics/Components/StructuralProximity.swift create mode 100644 Sources/CompletionScoring/Semantics/Components/SynchronicityCompatibility.swift create mode 100644 Sources/CompletionScoring/Semantics/Components/TypeCompatibility.swift create mode 100644 Sources/CompletionScoring/Semantics/SemanticClassification.swift create mode 100644 Sources/CompletionScoring/Text/CandidateBatch.swift create mode 100644 Sources/CompletionScoring/Text/InfluencingIdentifiers.swift create mode 100644 Sources/CompletionScoring/Text/MatchCollator.Match.swift create mode 100644 Sources/CompletionScoring/Text/MatchCollator.Selection.swift create mode 100644 Sources/CompletionScoring/Text/MatchCollator.swift create mode 100644 Sources/CompletionScoring/Text/Pattern.swift create mode 100644 Sources/CompletionScoring/Text/RejectionFilter.swift create mode 100644 Sources/CompletionScoring/Text/ScoredMatchSelector.swift create mode 100644 Sources/CompletionScoring/Text/UTF8Byte.swift create mode 100644 Sources/CompletionScoring/Utilities/SelectTopK.swift create mode 100644 Sources/CompletionScoring/Utilities/Serialization/BinaryCodable.swift create mode 100644 Sources/CompletionScoring/Utilities/Serialization/BinaryDecoder.swift create mode 100644 Sources/CompletionScoring/Utilities/Serialization/BinaryEncoder.swift create mode 100644 Sources/CompletionScoring/Utilities/Serialization/Conformances/Array+BinaryCodable.swift create mode 100644 Sources/CompletionScoring/Utilities/Serialization/Conformances/Dictionary+BinaryCodable.swift create mode 100644 Sources/CompletionScoring/Utilities/Serialization/Conformances/OptionSet+BinaryCodable.swift create mode 100644 Sources/CompletionScoring/Utilities/Serialization/Conformances/Optional+BinaryCodable.swift create mode 100644 Sources/CompletionScoring/Utilities/Serialization/Conformances/Scalars+BinaryCodable.swift create mode 100644 Sources/CompletionScoring/Utilities/Serialization/Conformances/String+BinaryCodable.swift create mode 100644 Sources/CompletionScoring/Utilities/SwiftExtensions.swift create mode 100644 Sources/CompletionScoring/Utilities/UnsafeArray.swift create mode 100644 Sources/CompletionScoring/Utilities/UnsafeStackAllocator.swift create mode 120000 Sources/CompletionScoringForPlugin create mode 100644 Sources/CompletionScoringTestSupport/CommonFunctionTerms.json create mode 100644 Sources/CompletionScoringTestSupport/RandomSeed.plist create mode 100644 Sources/CompletionScoringTestSupport/RepeatableRandomNumberGenerator.swift create mode 100644 Sources/CompletionScoringTestSupport/SymbolGenerator.swift create mode 100644 Sources/CompletionScoringTestSupport/TestExtensions.swift create mode 100644 Sources/CompletionScoringTestSupport/Timings.swift create mode 100644 Sources/CompletionScoringTestSupport/WeightedChoices.swift create mode 100644 Sources/Csourcekitd/include/CodeCompletionSwiftInterop.h create mode 100644 Sources/Csourcekitd/include/plugin.h create mode 120000 Sources/SKLoggingForPlugin create mode 120000 Sources/SKUtilitiesForPlugin create mode 120000 Sources/SourceKitDForPlugin create mode 120000 Sources/SwiftExtensionsForPlugin create mode 100644 Sources/SwiftSourceKitClientPlugin/ClientPlugin.swift create mode 100644 Sources/SwiftSourceKitPlugin/ASTCompletion/ASTCompletionItem.swift create mode 100644 Sources/SwiftSourceKitPlugin/ASTCompletion/CompletionSession.swift create mode 100644 Sources/SwiftSourceKitPlugin/CodeCompletion/Completion+Diagnostic.swift create mode 100644 Sources/SwiftSourceKitPlugin/CodeCompletion/Completion+Identifier.swift create mode 100644 Sources/SwiftSourceKitPlugin/CodeCompletion/Completion+ItemKind.swift create mode 100644 Sources/SwiftSourceKitPlugin/CodeCompletion/Completion+PriorityBucket.swift create mode 100644 Sources/SwiftSourceKitPlugin/CodeCompletion/Completion+SemanticContext.swift create mode 100644 Sources/SwiftSourceKitPlugin/CodeCompletion/Completion+TypeRelation.swift create mode 100644 Sources/SwiftSourceKitPlugin/CodeCompletion/CompletionContext.swift create mode 100644 Sources/SwiftSourceKitPlugin/CodeCompletion/CompletionItem.swift create mode 100644 Sources/SwiftSourceKitPlugin/CodeCompletion/CompletionOptions.swift create mode 100644 Sources/SwiftSourceKitPlugin/CodeCompletion/CompletionSorting.swift create mode 100644 Sources/SwiftSourceKitPlugin/CodeCompletion/Connection.swift create mode 100644 Sources/SwiftSourceKitPlugin/CodeCompletion/Location.swift create mode 100644 Sources/SwiftSourceKitPlugin/CodeCompletion/Position.swift create mode 100644 Sources/SwiftSourceKitPlugin/CodeCompletion/TextEdit.swift create mode 100644 Sources/SwiftSourceKitPlugin/CodeCompletion/WithArrayOfCStrings.swift create mode 100644 Sources/SwiftSourceKitPlugin/CompletionProvider.swift create mode 100644 Sources/SwiftSourceKitPlugin/CompletionResultsArrayBuilder.swift create mode 100644 Sources/SwiftSourceKitPlugin/Plugin.swift create mode 100644 Sources/SwiftSourceKitPlugin/SKDRequestArrayReader.swift create mode 100644 Sources/SwiftSourceKitPlugin/SKDRequestDictionaryReader.swift create mode 100644 Sources/SwiftSourceKitPlugin/SKDResponse.swift create mode 100644 Sources/SwiftSourceKitPlugin/SKDResponseArrayBuilder.swift create mode 100644 Sources/SwiftSourceKitPlugin/SKDResponseDictionaryBuilder.swift create mode 100644 Sources/SwiftSourceKitPlugin/SKDResponseValue.swift create mode 100644 Sources/SwiftSourceKitPlugin/SourceKitDWrappers.swift create mode 100644 Sources/SwiftSourceKitPlugin/SourceKitPluginError.swift create mode 100644 Sources/SwiftSourceKitPluginCommon/CompletionResultsArray.swift create mode 100644 Sources/SwiftSourceKitPluginCommon/DynamicallyLoadedSourceKitD+relativeToPlugin.swift create mode 100644 Tests/CompletionScoringPerfTests/CandidateBatchPerfTests.swift create mode 100644 Tests/CompletionScoringPerfTests/ScoringPerfTests.Completion.swift create mode 100644 Tests/CompletionScoringPerfTests/ScoringPerfTests.Corpus.swift create mode 100644 Tests/CompletionScoringPerfTests/ScoringPerfTests.Module.swift create mode 100644 Tests/CompletionScoringPerfTests/ScoringPerfTests.Provider.swift create mode 100644 Tests/CompletionScoringPerfTests/ScoringPerfTests.swift create mode 100644 Tests/CompletionScoringPerfTests/Supporting Files/CodeCompletionFoundationPerfTests-Info.plist create mode 100644 Tests/CompletionScoringTests/BinaryCodingTests.swift create mode 100644 Tests/CompletionScoringTests/CandidateBatchTests.swift create mode 100644 Tests/CompletionScoringTests/CompletionScoreTests.swift create mode 100644 Tests/CompletionScoringTests/InfluencingIdentifiersTests.swift create mode 100644 Tests/CompletionScoringTests/PatternTests.swift create mode 100644 Tests/CompletionScoringTests/RejectionFilterTests.swift create mode 100644 Tests/CompletionScoringTests/SemanticScoredText.swift create mode 100644 Tests/CompletionScoringTests/StackAllocateTests.swift create mode 100644 Tests/CompletionScoringTests/Supporting Files/CodeCompletionFoundationTests-Info.plist create mode 100644 Tests/CompletionScoringTests/SwiftExtensionsTests.swift create mode 100644 Tests/CompletionScoringTests/TestScoringWorkload.swift create mode 100644 Tests/CompletionScoringTests/TestTimings.swift create mode 100644 Tests/CompletionScoringTests/TopKTests.swift create mode 100644 Tests/CompletionScoringTests/UTF8ByteTests.swift create mode 100644 Tests/CompletionScoringTests/XCTestCaseScoringAdditions.swift create mode 100644 Tests/SwiftSourceKitPluginTests/PluginPaths.swift create mode 100644 Tests/SwiftSourceKitPluginTests/PluginSwiftPMTestProject.swift create mode 100644 Tests/SwiftSourceKitPluginTests/SwiftSourceKitPluginTests.swift diff --git a/Package.swift b/Package.swift index 2d87941e6..00515fd01 100644 --- a/Package.swift +++ b/Package.swift @@ -13,6 +13,8 @@ var products: [Product] = [ .executable(name: "sourcekit-lsp", targets: ["sourcekit-lsp"]), .library(name: "_SourceKitLSP", targets: ["SourceKitLSP"]), .library(name: "LSPBindings", targets: ["LanguageServerProtocol", "LanguageServerProtocolJSONRPC"]), + .library(name: "SwiftSourceKitPlugin", type: .dynamic, targets: ["SwiftSourceKitPlugin"]), + .library(name: "SwiftSourceKitClientPlugin", type: .dynamic, targets: ["SwiftSourceKitClientPlugin"]), ] var targets: [Target] = [ @@ -109,6 +111,40 @@ var targets: [Target] = [ dependencies: [] ), + // MARK: CompletionScoring + + .target( + name: "CompletionScoring", + dependencies: [], + swiftSettings: globalSwiftSettings + ), + + .target( + name: "CompletionScoringForPlugin", + dependencies: [], + swiftSettings: globalSwiftSettings + ), + + .testTarget( + name: "CompletionScoringTests", + dependencies: ["CompletionScoring", "CompletionScoringTestSupport", "SwiftExtensions"], + swiftSettings: globalSwiftSettings + ), + + .testTarget( + name: "CompletionScoringPerfTests", + dependencies: ["CompletionScoring", "CompletionScoringTestSupport", "SwiftExtensions"], + swiftSettings: globalSwiftSettings + ), + + // MARK: CompletionScoringTestSupport + + .target( + name: "CompletionScoringTestSupport", + dependencies: ["CompletionScoring", "SwiftExtensions"], + swiftSettings: globalSwiftSettings + ), + // MARK: CSKTestSupport .target( @@ -273,6 +309,21 @@ var targets: [Target] = [ swiftSettings: globalSwiftSettings + lspLoggingSwiftSettings ), + .target( + name: "SKLoggingForPlugin", + dependencies: [ + "SwiftExtensionsForPlugin" + ], + exclude: ["CMakeLists.txt"], + swiftSettings: globalSwiftSettings + lspLoggingSwiftSettings + [ + // We can't depend on swift-crypto in the plugin because we can't module-alias it due to https://github.com/swiftlang/swift-package-manager/issues/8119 + .define("NO_CRYPTO_DEPENDENCY"), + .unsafeFlags([ + "-module-alias", "SwiftExtensions=SwiftExtensionsForPlugin", + ]), + ] + ), + .testTarget( name: "SKLoggingTests", dependencies: [ @@ -308,6 +359,21 @@ var targets: [Target] = [ swiftSettings: globalSwiftSettings ), + .target( + name: "SKUtilitiesForPlugin", + dependencies: [ + "SKLoggingForPlugin", + "SwiftExtensionsForPlugin", + ], + exclude: ["CMakeLists.txt"], + swiftSettings: globalSwiftSettings + [ + .unsafeFlags([ + "-module-alias", "SKLogging=SKLoggingForPlugin", + "-module-alias", "SwiftExtensions=SwiftExtensionsForPlugin", + ]) + ] + ), + .testTarget( name: "SKUtilitiesTests", dependencies: [ @@ -354,6 +420,22 @@ var targets: [Target] = [ swiftSettings: globalSwiftSettings ), + .target( + name: "SourceKitDForPlugin", + dependencies: [ + "Csourcekitd", + "SKLoggingForPlugin", + "SwiftExtensionsForPlugin", + ], + exclude: ["CMakeLists.txt", "sourcekitd_uids.swift.gyb"], + swiftSettings: globalSwiftSettings + [ + .unsafeFlags([ + "-module-alias", "SKLogging=SKLoggingForPlugin", + "-module-alias", "SwiftExtensions=SwiftExtensionsForPlugin", + ]) + ] + ), + .testTarget( name: "SourceKitDTests", dependencies: [ @@ -429,6 +511,13 @@ var targets: [Target] = [ swiftSettings: globalSwiftSettings ), + .target( + name: "SwiftExtensionsForPlugin", + dependencies: ["CAtomics"], + exclude: ["CMakeLists.txt"], + swiftSettings: globalSwiftSettings + ), + .testTarget( name: "SwiftExtensionsTests", dependencies: [ @@ -439,6 +528,80 @@ var targets: [Target] = [ swiftSettings: globalSwiftSettings ), + // MARK: SwiftSourceKitClientPlugin + + .target( + name: "SwiftSourceKitClientPlugin", + dependencies: [ + "Csourcekitd", + "SourceKitDForPlugin", + "SwiftSourceKitPluginCommon", + ], + swiftSettings: globalSwiftSettings + [ + .unsafeFlags([ + "-module-alias", "SourceKitD=SourceKitDForPlugin", + ]) + ] + ), + + // MARK: SwiftSourceKitPluginCommon + + .target( + name: "SwiftSourceKitPluginCommon", + dependencies: [ + "Csourcekitd", + "SourceKitDForPlugin", + "SwiftExtensionsForPlugin", + "SKLoggingForPlugin", + ], + swiftSettings: globalSwiftSettings + [ + .unsafeFlags([ + "-module-alias", "SourceKitD=SourceKitDForPlugin", + "-module-alias", "SwiftExtensions=SwiftExtensionsForPlugin", + "-module-alias", "SKLogging=SKLoggingForPlugin", + ]) + ] + ), + + // MARK: SwiftSourceKitPlugin + + .target( + name: "SwiftSourceKitPlugin", + dependencies: [ + "Csourcekitd", + "CompletionScoringForPlugin", + "SKUtilitiesForPlugin", + "SKLoggingForPlugin", + "SourceKitDForPlugin", + "SwiftSourceKitPluginCommon", + "SwiftExtensionsForPlugin", + ], + swiftSettings: globalSwiftSettings + [ + .unsafeFlags([ + "-module-alias", "CompletionScoring=CompletionScoringForPlugin", + "-module-alias", "SKUtilities=SKUtilitiesForPlugin", + "-module-alias", "SourceKitD=SourceKitDForPlugin", + "-module-alias", "SKLogging=SKLoggingForPlugin", + "-module-alias", "SwiftExtensions=SwiftExtensionsForPlugin", + ]) + ] + ), + + .testTarget( + name: "SwiftSourceKitPluginTests", + dependencies: [ + "BuildSystemIntegration", + "CompletionScoring", + "Csourcekitd", + "LanguageServerProtocol", + "SKTestSupport", + "SourceKitD", + "SwiftExtensions", + "ToolchainRegistry", + ], + swiftSettings: globalSwiftSettings + ), + // MARK: ToolchainRegistry .target( diff --git a/Sources/CompletionScoring/Semantics/CompletionScore.swift b/Sources/CompletionScoring/Semantics/CompletionScore.swift new file mode 100644 index 000000000..8affba6e8 --- /dev/null +++ b/Sources/CompletionScoring/Semantics/CompletionScore.swift @@ -0,0 +1,75 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +/// Represents a composite score formed from a semantic and textual components. +/// +/// The textual component forms the bulk of the score typically having a value in the 10's to 100's. +/// You can think of the semantic component as a bonus to the text score. +/// It usually has a value between 0 and 2, and is used as a multiplier. +package struct CompletionScore: Comparable { + package var semanticComponent: Double + package var textComponent: Double + + package init(textComponent: Double, semanticComponent: Double) { + self.semanticComponent = semanticComponent + self.textComponent = textComponent + } + + package init(textComponent: Double, semanticClassification: SemanticClassification) { + self.semanticComponent = semanticClassification.score + self.textComponent = textComponent + } + + package var value: Double { + semanticComponent * textComponent + } + + package static func < (_ lhs: Self, _ rhs: Self) -> Bool { + lhs.value < rhs.value + } +} + +// MARK: - Deprecated - +extension CompletionScore { + /// There is no natural order to these arguments, so they're alphabetical. + @available( + *, + deprecated, + renamed: + "SemanticClassification(completionKind:deprecationStatus:flair:moduleProximity:popularity:scopeProximity:structuralProximity:synchronicityCompatibility:typeCompatibility:)" + ) + package static func semanticScore( + completionKind: CompletionKind, + deprecationStatus: DeprecationStatus, + flair: Flair, + moduleProximity: ModuleProximity, + popularity: Popularity, + scopeProximity: ScopeProximity, + structuralProximity: StructuralProximity, + synchronicityCompatibility: SynchronicityCompatibility, + typeCompatibility: TypeCompatibility + ) -> Double { + SemanticClassification( + availability: deprecationStatus, + completionKind: completionKind, + flair: flair, + moduleProximity: moduleProximity, + popularity: popularity, + scopeProximity: scopeProximity, + structuralProximity: structuralProximity, + synchronicityCompatibility: synchronicityCompatibility, + typeCompatibility: typeCompatibility + ).score + } +} diff --git a/Sources/CompletionScoring/Semantics/Components/Availability.swift b/Sources/CompletionScoring/Semantics/Components/Availability.swift new file mode 100644 index 000000000..8a45de790 --- /dev/null +++ b/Sources/CompletionScoring/Semantics/Components/Availability.swift @@ -0,0 +1,75 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +package enum Availability: Equatable { + /// Example: Either not tagged, or explicit availability is compatible with current build context + case available + + /// Example: Explicitly unavailable in current build context - ie, only for another platform. + case unavailable + + /// Example: deprecated in the future + case softDeprecated + + /// Example: deprecated in the present, or past + case deprecated + + /// Completion provider doesn't know if the method is deprecated or not + case unknown + + /// Example: keyword + case inapplicable + + /// Example: Provider was written before this enum existed, and didn't have an opportunity to provide a value + case unspecified +} + +extension Availability: BinaryCodable { + package init(_ decoder: inout BinaryDecoder) throws { + self = try decoder.decodeEnumByte { decoder, n in + switch n { + case 0: return .available + case 1: return .unavailable + case 2: return .softDeprecated + case 3: return .deprecated + case 4: return .unknown + case 5: return .inapplicable + case 6: return .unspecified + default: return nil + } + } + } + + package func encode(_ encoder: inout BinaryEncoder) { + let value: UInt8 + switch self { + case .available: value = 0 + case .unavailable: value = 1 + case .softDeprecated: value = 2 + case .deprecated: value = 3 + case .unknown: value = 4 + case .inapplicable: value = 5 + case .unspecified: value = 6 + } + encoder.write(value) + } +} + +@available(*, deprecated, renamed: "Availability") +package typealias DeprecationStatus = Availability + +extension Availability { + @available(*, deprecated, renamed: "Availability.available") + package static let none = DeprecationStatus.available +} diff --git a/Sources/CompletionScoring/Semantics/Components/CompletionKind.swift b/Sources/CompletionScoring/Semantics/Components/CompletionKind.swift new file mode 100644 index 000000000..105fff56a --- /dev/null +++ b/Sources/CompletionScoring/Semantics/Components/CompletionKind.swift @@ -0,0 +1,92 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +package enum CompletionKind: Equatable { + /// Example: try, throw, where + case keyword + + /// Example: A case in an enumeration + case enumCase + + /// Example: A local, argument, property within a type + case variable + + /// Example: A function at global scope, or within some other type. + case function + + /// Example: An init method + case initializer + + /// Example: in `append(|)`, suggesting `contentsOf:` + case argumentLabels + + /// Example: `String`, `Int`, generic parameter, typealias, associatedtype + case type + + /// Example: A `guard let` template for a local variable. + case template + + /// Example: Foundation, AppKit, UIKit, SwiftUI + case module + + /// Example: Something not listed here, consider adding a new case + case other + + /// Example: Completion provider can't even tell what it's offering up. + case unknown + + /// Example: Provider was written before this enum existed, and didn't have an opportunity to provide a value + case unspecified +} + +extension CompletionKind: BinaryCodable { + package init(_ decoder: inout BinaryDecoder) throws { + self = try decoder.decodeEnumByte { decoder, n in + switch n { + case 0: return .keyword + case 1: return .enumCase + case 2: return .variable + case 3: return .function + case 4: return .initializer + case 5: return .argumentLabels + case 6: return .type + case 7: return .template + case 8: return .other + case 9: return .unknown + case 10: return .unspecified + case 11: return .module + default: return nil + } + } + } + + package func encode(_ encoder: inout BinaryEncoder) { + let value: UInt8 + switch self { + case .keyword: value = 0 + case .enumCase: value = 1 + case .variable: value = 2 + case .function: value = 3 + case .initializer: value = 4 + case .argumentLabels: value = 5 + case .type: value = 6 + case .template: value = 7 + case .other: value = 8 + case .unknown: value = 9 + case .unspecified: value = 10 + case .module: value = 11 + } + encoder.write(value) + } +} diff --git a/Sources/CompletionScoring/Semantics/Components/Flair.swift b/Sources/CompletionScoring/Semantics/Components/Flair.swift new file mode 100644 index 000000000..0a36d31d0 --- /dev/null +++ b/Sources/CompletionScoring/Semantics/Components/Flair.swift @@ -0,0 +1,139 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +package struct Flair: OptionSet { + package var rawValue: Int64 + + package init(rawValue: Int64) { + self.rawValue = rawValue + } + + package init(_ rawValue: Int64) { + self.rawValue = rawValue + } + + /// Xcode prior to version 13, grouped many high priority completions under the tag 'expression specific'. + /// To aid in mapping those into the new API, redefine this case as a catch-all 'high priority' expression. + /// Please do not use it for new cases, instead, add a case to this enum to model the significance of the completion. + package static let oldExpressionSpecific_pleaseAddSpecificCaseToThisEnum = Flair(1 << 0) + + /// E.g. `override func foo() { super.foo() ...` + package static let chainedCallToSuper = Flair(1 << 1) + + /// E.g. `bar.baz` in `foo.bar.baz`. + package static let chainedMember = Flair(1 << 2) + + /// E.g. using class, struct, protocol, public, private, or fileprivate at file level. + internal static let _situationallyLikely = Flair(1 << 3) + + /// E.g. using `class` inside of a function body, or a protocol in an expression. + internal static let _situationallyUnlikely = Flair(1 << 4) + + /// E.g. referencing a type at top level position in a non script/main.swift file. + internal static let _situationallyInvalid = Flair(1 << 5) + + /// E.g. an instance method/property in SwiftUI ViewBuilder with implicit-self context. + package static let swiftUIModifierOnSelfWhileBuildingSelf = Flair(1 << 6) + + /// E.g. `frame()`, since you almost certainly want to pass an argument to it. + package static let swiftUIUnlikelyViewMember = Flair(1 << 7) + + /// E.g. using struct, enum, protocol, class, public, private, or fileprivate at file level. + package static let commonKeywordAtCurrentPosition = Flair(1 << 8) + + /// E.g. nesting class in a function. + package static let rareKeywordAtCurrentPosition = Flair(1 << 9) + + /// E.g. using a protocol by name in an expression in a non-type position. `let x = 3 + Comparable…` + package static let rareTypeAtCurrentPosition = Flair(1 << 10) + + /// E.g. referencing a type, function, etc… at top level position in a non script/main.swift file. + package static let expressionAtNonScriptOrMainFileScope = Flair(1 << 11) + + /// E.g. `printContents`, which is almost never what you want when typing `print`. + package static let rareMemberWithCommonName = Flair(1 << 12) + + @available( + *, + deprecated, + message: """ + This is an escape hatch for scenarios we haven't thought of. \ + When using, file a bug report to name the new situational oddity so that we can directly model it. + """ + ) + package static let situationallyLikely = _situationallyLikely + + @available( + *, + deprecated, + message: """ + This is an escape hatch for scenarios we haven't thought of. \ + When using, file a bug report to name the new situational oddity so that we can directly model it. + """ + ) + package static let situationallyUnlikely = _situationallyUnlikely + + @available( + *, + deprecated, + message: """ + This is an escape hatch for scenarios we haven't thought of. \ + When using, file a bug report to name the new situational oddity so that we can directly model it. + """ + ) + package static let situationallyInvalid = _situationallyInvalid +} + +extension Flair: CustomDebugStringConvertible { + private static let namedValues: [(Flair, String)] = [ + ( + .oldExpressionSpecific_pleaseAddSpecificCaseToThisEnum, + "oldExpressionSpecific_pleaseAddSpecificCaseToThisEnum" + ), + (.chainedCallToSuper, "chainedCallToSuper"), + (.chainedMember, "chainedMember"), + (._situationallyLikely, "_situationallyLikely"), + (._situationallyUnlikely, "_situationallyUnlikely"), + (._situationallyInvalid, "_situationallyInvalid"), + (.swiftUIModifierOnSelfWhileBuildingSelf, "swiftUIModifierOnSelfWhileBuildingSelf"), + (.swiftUIUnlikelyViewMember, "swiftUIUnlikelyViewMember"), + (.commonKeywordAtCurrentPosition, "commonKeywordAtCurrentPosition"), + (.rareKeywordAtCurrentPosition, "rareKeywordAtCurrentPosition"), + (.rareTypeAtCurrentPosition, "rareTypeAtCurrentPosition"), + (.expressionAtNonScriptOrMainFileScope, "expressionAtNonScriptOrMainFileScope"), + (.rareMemberWithCommonName, "rareMemberWithCommonName"), + ] + + package var debugDescription: String { + var descriptions = [String]() + for (flair, name) in Self.namedValues { + if self.contains(flair) { + descriptions.append(name) + } + } + if descriptions.isEmpty { + return "none" + } else { + return descriptions.joined(separator: ",") + } + } +} + +extension Flair: OptionSetBinaryCodable {} + +extension Flair { + package var factor: Double { + return self.scoreComponent + } +} diff --git a/Sources/CompletionScoring/Semantics/Components/ModuleProximity.swift b/Sources/CompletionScoring/Semantics/Components/ModuleProximity.swift new file mode 100644 index 000000000..5da20ac74 --- /dev/null +++ b/Sources/CompletionScoring/Semantics/Components/ModuleProximity.swift @@ -0,0 +1,71 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +package enum ModuleProximity: Equatable { + /// Example: Distance 0 is this module. if you have only "import AppKit", AppKit would be 1, Foundation and + /// CoreGraphics would be 2. + case imported(distance: Int) + + /// Example: Referencing NSDocument (AppKit) from a tool only using Foundation. An import to (AppKit) would have to + /// be added. + case importable + + /// Example: Keywords + case inapplicable + + // Completion provider doesn't understand modules + case unknown + + /// Example: Circular dependency, wrong platform + case invalid + + /// Example: Provider was written before this enum existed, and didn't have an opportunity to provide a value + case unspecified + + package static let same = ModuleProximity.imported(distance: 0) +} + +extension ModuleProximity: BinaryCodable { + package init(_ decoder: inout BinaryDecoder) throws { + self = try decoder.decodeEnumByte { decoder, n in + switch n { + case 0: return .imported(distance: try Int(&decoder)) + case 1: return .importable + case 2: return .inapplicable + case 3: return .unknown + case 4: return .invalid + case 5: return .unspecified + default: return nil + } + } + } + + package func encode(_ encoder: inout BinaryEncoder) { + switch self { + case .imported(let distance): + encoder.writeByte(0) + encoder.write(distance) + case .importable: + encoder.writeByte(1) + case .inapplicable: + encoder.writeByte(2) + case .unknown: + encoder.writeByte(3) + case .invalid: + encoder.writeByte(4) + case .unspecified: + encoder.writeByte(5) + } + } +} diff --git a/Sources/CompletionScoring/Semantics/Components/Popularity.swift b/Sources/CompletionScoring/Semantics/Components/Popularity.swift new file mode 100644 index 000000000..67af9215a --- /dev/null +++ b/Sources/CompletionScoring/Semantics/Components/Popularity.swift @@ -0,0 +1,125 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +package struct PopularityScoreComponent: Equatable, Comparable { + var value: Double + static let unspecified = PopularityScoreComponent(value: unspecifiedScore) + static let none = PopularityScoreComponent(value: 1.0) + + package static func < (_ lhs: Self, _ rhs: Self) -> Bool { + lhs.value < rhs.value + } +} + +extension PopularityScoreComponent: BinaryCodable { + package init(_ decoder: inout BinaryDecoder) throws { + value = try Double(&decoder) + } + + package func encode(_ encoder: inout BinaryEncoder) { + encoder.write(value) + } +} + +package struct Popularity: Equatable, Comparable { + package var scoreComponent: Double { + symbolComponent * moduleComponent + } + + package var symbolComponent: Double + + package var moduleComponent: Double + + // TODO: remove once `PopularityTable` is removed. + package static let unspecified = Popularity(scoreComponent: unspecifiedScore) + package static let none = Popularity(scoreComponent: 1.0) + + package enum Category { + /// Used by `PopularityIndex`, where the popularities don't have much context. + case index + + /// Used when a client has a lot context. Allowing for a more precise popularity boost. + case predictive + + private static let predictiveMin = 1.10 + private static let predictiveMax = 1.30 + + var minimum: Double { + switch self { + case .index: + return 1.02 + case .predictive: + return Self.predictiveMin + } + } + + var maximum: Double { + switch self { + case .index: + return 1.10 + case .predictive: + return Self.predictiveMax + } + } + } + + @available(*, deprecated) + package init(probability: Double) { + self.init(probability: probability, category: .index) + } + + /// - Parameter probability: a value in range `0...1` + package init(probability: Double, category: Category) { + let score = Self.scoreComponent(probability: probability, category: category) + self.init(scoreComponent: score) + } + + /// Takes value in range `0...1`, + /// and converts to a value that can be used for multiplying with other score components. + static func scoreComponent(probability: Double, category: Category) -> Double { + let min = category.minimum + let max = category.maximum + if min > max { + assertionFailure("min \(min) > max \(max)") + return 1.0 + } + return (probability * (max - min)) + min + } + + package init(scoreComponent: Double) { + self.symbolComponent = scoreComponent + self.moduleComponent = 1.0 + } + + internal init(symbolComponent: Double, moduleComponent: Double) { + self.symbolComponent = symbolComponent + self.moduleComponent = moduleComponent + } + + package static func < (_ lhs: Self, _ rhs: Self) -> Bool { + lhs.scoreComponent < rhs.scoreComponent + } +} + +extension Popularity: BinaryCodable { + package init(_ decoder: inout BinaryDecoder) throws { + symbolComponent = try Double(&decoder) + moduleComponent = try Double(&decoder) + } + + package func encode(_ encoder: inout BinaryEncoder) { + encoder.write(symbolComponent) + encoder.write(moduleComponent) + } +} diff --git a/Sources/CompletionScoring/Semantics/Components/PopularityIndex.swift b/Sources/CompletionScoring/Semantics/Components/PopularityIndex.swift new file mode 100644 index 000000000..cb4e45078 --- /dev/null +++ b/Sources/CompletionScoring/Semantics/Components/PopularityIndex.swift @@ -0,0 +1,221 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +/// A `PopularityIndex` is constructed from symbol reference frequencies and uses that data to bestow +/// `Popularity` bonuses on completions. +package struct PopularityIndex { + + /// The namespace of a symbol. + /// + /// Examples + /// * `Swift.Array.append(:)` would be `Scope(container: "Array", module: "Swift")` + /// * `Swift.Array` would be `Scope(container: nil, module: "Swift")`. + /// + /// This library imposes no constraints on formatting `container`. It's entirely up to the client to + /// decide how precise to be, and how to spell values. They could use `[String]`, `Array` + /// or `Array`. It only matters that they refer to types consistently. They're also free to model + /// inner types with strings like `List.Node`. + package struct Scope: Hashable { + package var container: String? + package var module: String + + package init(container: String?, module: String) { + self.module = module + self.container = container + } + } + + /// A name within a scope. + /// + /// Examples + /// * `Swift.Array.append(:)` would be: + /// * `Symbol(name: "append(:)", scope: Scope(container: "Array", module: "Swift"))` + /// * `Swift.Array` would be: + /// * `Symbol(name: "Array", scope: Scope(container: nil, module: "Swift"))` + /// + /// This library imposes no constraints on formatting `name`. It's entirely up to the client to use + /// consistent values. For example, they could independently track overloads by including types + /// in function names, or they could combine all related methods by tracking only function base + /// names. + package struct Symbol: Hashable { + package var name: String + package var scope: Scope + + package init(name: String, scope: Scope) { + self.name = name + self.scope = scope + } + } + + package private(set) var symbolPopularity: [Symbol: PopularityScoreComponent] = [:] + package private(set) var modulePopularity: [String: PopularityScoreComponent] = [:] + + private var knownScopes = Set() + + /// Clients can use this to find a relevant `Scope`. + /// To contruct a `Symbol` to pass to `popularity(of:)`. + package func isKnownScope(_ scope: Scope) -> Bool { + return knownScopes.contains(scope) + } + + /// - Parameters: + /// - `symbolReferencePercentages`: Symbol reference percentages per scope. + /// For example, if the data that produced the symbol reference percentags had 1 call to `Array.append(:)`, + /// 3 calls to `Array.count`, and 1 call to `String.append(:)` the table would be: + /// ``` + /// [ + /// "Swift.Array" : [ + /// "append(:)" : 0.25, + /// "count" : 0.75 + /// ], + /// "Swift.String" : [ + /// "append(:)" : 1.0 + /// ] + /// ] + /// ``` + /// - `notoriousSymbols`: Symbols from this list will get a significant penalty. + /// - `popularModules`: Symbols from these modules will get a slight bonus. + /// - `notoriousModules`: symbols from these modules will get a significant penalty. + package init( + symbolReferencePercentages: [Scope: [String: Double]], + notoriousSymbols: [Symbol], + popularModules: [String], + notoriousModules: [String] + ) { + knownScopes = Set(symbolReferencePercentages.keys) + + raisePopularities(symbolReferencePercentages: symbolReferencePercentages) + raisePopularities(popularModules: popularModules) + + // Even if data shows that it's popular, if we manually penalized it, always do that. + lowerPopularities(notoriousModules: notoriousModules) + lowerPopularities(notoriousSymbols: notoriousSymbols) + } + + fileprivate init() {} + + private mutating func raisePopularities(symbolReferencePercentages: [Scope: [String: Double]]) { + for (scope, namedReferencePercentages) in symbolReferencePercentages { + if let maxReferencePercentage = namedReferencePercentages.lazy.map(\.value).max() { + for (completion, referencePercentage) in namedReferencePercentages { + let symbol = Symbol(name: completion, scope: scope) + let normalizedScore = referencePercentage / maxReferencePercentage // 0...1 + let flattenedScore = pow(normalizedScore, 0.25) // Don't make it so much of a winner takes all + symbolPopularity.raise( + symbol, + toAtLeast: Popularity.scoreComponent(probability: flattenedScore, category: .index) + ) + } + } + } + } + + private mutating func lowerPopularities(notoriousSymbols: [Symbol]) { + symbolPopularity.lower(notoriousSymbols, toAtMost: Availability.deprecated.scoreComponent) + } + + private mutating func lowerPopularities(notoriousModules: [String]) { + modulePopularity.lower(notoriousModules, toAtMost: Availability.deprecated.scoreComponent) + } + + private mutating func raisePopularities(popularModules: [String]) { + modulePopularity.raise(popularModules, toAtLeast: Popularity.scoreComponent(probability: 0.0, category: .index)) + } + + package func popularity(of symbol: Symbol) -> Popularity { + let symbolPopularity = symbolPopularity[symbol] ?? .none + let modulePopularity = modulePopularity[symbol.scope.module] ?? .none + return Popularity(symbolComponent: symbolPopularity.value, moduleComponent: modulePopularity.value) + } +} + +fileprivate extension Dictionary where Value == PopularityScoreComponent { + mutating func raise(_ key: Key, toAtLeast minimum: Double) { + let leastPopular = PopularityScoreComponent(value: -Double.infinity) + if self[key, default: leastPopular].value < minimum { + self[key] = PopularityScoreComponent(value: minimum) + } + } + + mutating func lower(_ key: Key, toAtMost maximum: Double) { + let mostPopular = PopularityScoreComponent(value: Double.infinity) + if self[key, default: mostPopular].value > maximum { + self[key] = PopularityScoreComponent(value: maximum) + } + } + + mutating func raise(_ keys: [Key], toAtLeast minimum: Double) { + for key in keys { + raise(key, toAtLeast: minimum) + } + } + + mutating func lower(_ keys: [Key], toAtMost maximum: Double) { + for key in keys { + lower(key, toAtMost: maximum) + } + } +} + +/// Implement coding with BinaryCodable without singing up for package conformance +extension PopularityIndex { + package enum SerializationVersion: Int { + case initial + } + + private struct SerializableSymbol: Hashable, BinaryCodable { + var symbol: Symbol + + init(symbol: Symbol) { + self.symbol = symbol + } + + init(_ decoder: inout BinaryDecoder) throws { + let name = try String(&decoder) + let container = try String?(&decoder) + let module = try String(&decoder) + symbol = Symbol(name: name, scope: Scope(container: container, module: module)) + } + + func encode(_ encoder: inout BinaryEncoder) { + encoder.write(symbol.name) + encoder.write(symbol.scope.container) + encoder.write(symbol.scope.module) + } + } + + package func serialize(version: SerializationVersion) -> [UInt8] { + BinaryEncoder.encode(contentVersion: version.rawValue) { encoder in + encoder.write(symbolPopularity.mapKeys(overwritingDuplicates: .affirmative, SerializableSymbol.init)) + encoder.write(modulePopularity) + } + } + + package static func deserialize(data serialization: [UInt8]) throws -> Self { + try BinaryDecoder.decode(bytes: serialization) { decoder in + switch SerializationVersion(rawValue: decoder.contentVersion) { + case .initial: + var index = Self() + index.symbolPopularity = try [SerializableSymbol: PopularityScoreComponent](&decoder).mapKeys( + overwritingDuplicates: .affirmative, + \.symbol + ) + index.modulePopularity = try [String: PopularityScoreComponent](&decoder) + return index + case .none: + throw GenericError("Unknown \(String(describing: self)) serialization format") + } + } + } +} diff --git a/Sources/CompletionScoring/Semantics/Components/PopularityTable.swift b/Sources/CompletionScoring/Semantics/Components/PopularityTable.swift new file mode 100644 index 000000000..60e94198f --- /dev/null +++ b/Sources/CompletionScoring/Semantics/Components/PopularityTable.swift @@ -0,0 +1,192 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +// TODO: deprecate +//@available(*, deprecated, message: "Use PopularityIndex instead.") +package struct PopularityTable { + /// Represents a list of symbols form a module, with a value for each symbol representing what % of references to this module were to that symbol. + package struct ModuleSymbolReferenceTable { + var symbolReferencePercentages: [String: Double] + var notoriousSymbols: [String] + package init(symbolReferencePercentages: [String: Double], notoriousSymbols: [String] = []) { + self.symbolReferencePercentages = symbolReferencePercentages + self.notoriousSymbols = notoriousSymbols + } + } + + /// package so `PopularityTable` can be serialized into an XPC object to be sent to the SourceKit plugin + package private(set) var symbolPopularity: [String: Popularity] = [:] + package private(set) var modulePopularity: [String: Popularity] = [:] + + /// Only to be used in the SourceKit plugin when deserializing a `PopularityTable` from an XPC object. + package init(symbolPopularity: [String: Popularity], modulePopularity: [String: Popularity]) { + self.symbolPopularity = symbolPopularity + self.modulePopularity = modulePopularity + } + + /// Initialize with popular symbol usage statistics, a list of recent completions, and module notoriety. + /// - Parameters: + /// - moduleSymbolReferenceTables: The symbol reference statistics should include the popular symbols for each + /// module. + /// - recentCompletions: A list of recent completions, with repetitions, with the earliest completions at the head + /// of the list. + /// - popularModules: Symbols from these modules will get a slight bonus. + /// - notoriousModules: symbols from these modules will get a significant penalty. + package init( + moduleSymbolReferenceTables: [ModuleSymbolReferenceTable], + recentCompletions: [String], + popularModules: [String], + notoriousModules: [String] + ) { + recordPopularSymbolsBonuses(modules: moduleSymbolReferenceTables) + recordNotoriousSymbolsBonuses(modules: moduleSymbolReferenceTables) + recordSymbolRecencyBonuses(recentCompletions: recentCompletions) + recordPopularModules(popularModules: popularModules) + recordNotoriousModules(notoriousModules: notoriousModules) + } + + /// Takes value from 0...1 + private func scoreComponent(normalizedPopularity: Double) -> Double { + let maxPopularityBonus = 1.10 + let minPopularityBonus = 1.02 + return (normalizedPopularity * (maxPopularityBonus - minPopularityBonus)) + minPopularityBonus + } + + private mutating func recordPopularSymbolsBonuses(modules: [ModuleSymbolReferenceTable]) { + for module in modules { + if let maxReferencePercentage = module.symbolReferencePercentages.map(\.value).max() { + for (symbol, referencePercentage) in module.symbolReferencePercentages { + let normalizedScore = referencePercentage / maxReferencePercentage // 0...1 + let flattenedScore = pow(normalizedScore, 0.25) // Don't make it so much of a winner takes all + symbolPopularity.record( + scoreComponent: scoreComponent(normalizedPopularity: flattenedScore), + for: symbol + ) + } + } + } + } + + /// Record recency so that repeated use also impacts score. Completing `NSString` 100 times, then completing + /// `NSStream` once should not make `NSStream` the top result. + private mutating func recordSymbolRecencyBonuses(recentCompletions: [String]) { + var pointsPerFilterText: [String: Int] = [:] + let count = recentCompletions.count + var totalPoints = 0 + for (position, filterText) in recentCompletions.enumerated() { + let points = count - position + pointsPerFilterText[filterText, default: 0] += points + totalPoints += points + } + for (filterText, points) in pointsPerFilterText { + let bonus = scoreComponent(normalizedPopularity: Double(points) / Double(totalPoints)) + symbolPopularity.record(scoreComponent: bonus, for: filterText) + } + } + + internal mutating func recordPopularModules(popularModules: [String]) { + let scoreComponent = scoreComponent(normalizedPopularity: 0.0) + for module in popularModules { + modulePopularity.record(scoreComponent: scoreComponent, for: module) + } + } + + internal mutating func recordNotoriousModules(notoriousModules: [String]) { + for module in notoriousModules { + modulePopularity.record(scoreComponent: Availability.deprecated.scoreComponent, for: module) + } + } + + internal mutating func record(notoriousSymbols: [String]) { + for symbol in notoriousSymbols { + symbolPopularity.record(scoreComponent: Availability.deprecated.scoreComponent, for: symbol) + } + } + + private mutating func recordNotoriousSymbolsBonuses(modules: [ModuleSymbolReferenceTable]) { + for module in modules { + record(notoriousSymbols: module.notoriousSymbols) + } + } + + private func popularity(symbol: String) -> Popularity { + return symbolPopularity[symbol] ?? .none + } + + private func popularity(module: String) -> Popularity { + return modulePopularity[module] ?? .none + } + + package func popularity(symbol: String?, module: String?) -> Popularity { + let symbolPopularity = symbol.map { popularity(symbol: $0) } ?? .none + let modulePopularity = module.map { popularity(module: $0) } ?? .none + return Popularity( + symbolComponent: symbolPopularity.scoreComponent, + moduleComponent: modulePopularity.scoreComponent + ) + } +} + +extension Dictionary { + fileprivate mutating func record(scoreComponent: Double, for key: String) { + let leastPopular = Popularity(scoreComponent: -Double.infinity) + if self[key, default: leastPopular].scoreComponent < scoreComponent { + self[key] = Popularity(scoreComponent: scoreComponent) + } + } +} + +// TODO: deprecate +//@available(*, deprecated, message: "Use PopularityIndex instead.") +extension PopularityTable { + package init(popularSymbols: [String] = [], recentSymbols: [String] = [], notoriousSymbols: [String] = []) { + add(popularSymbols: popularSymbols) + recordSymbolRecencyBonuses(recentCompletions: recentSymbols) + record(notoriousSymbols: notoriousSymbols) + } + + package mutating func add(popularSymbols: [String]) { + for (index, symbol) in popularSymbols.enumerated() { + let popularity = (1.0 - (Double(index) / Double(popularSymbols.count + 1))) // 1.0...0.0 + let scoreComponent = scoreComponent(normalizedPopularity: popularity) + symbolPopularity.record(scoreComponent: scoreComponent, for: symbol) + } + } +} + +// TODO: deprecate +//@available(*, deprecated, message: "Use PopularityIndex instead.") +extension PopularityTable { + @available( + *, + renamed: "ModuleSymbolReferenceTable", + message: "Popularity is now for modules in addition to symbols. This was renamed to be more precise." + ) + package typealias ModulePopularityTable = ModuleSymbolReferenceTable + + @available(*, deprecated, message: "Pass a module name with popularity(symbol:module:)") + package func popularity(for symbol: String) -> Popularity { + popularity(symbol: symbol, module: nil) + } + + @available(*, deprecated, message: "Pass popularModules: and notoriousModules:") + package init(modules: [ModuleSymbolReferenceTable], recentCompletions: [String]) { + self.init( + moduleSymbolReferenceTables: modules, + recentCompletions: recentCompletions, + popularModules: [], + notoriousModules: [] + ) + } +} diff --git a/Sources/CompletionScoring/Semantics/Components/ScopeProximity.swift b/Sources/CompletionScoring/Semantics/Components/ScopeProximity.swift new file mode 100644 index 000000000..24dca845d --- /dev/null +++ b/Sources/CompletionScoring/Semantics/Components/ScopeProximity.swift @@ -0,0 +1,81 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +package enum ScopeProximity: Equatable { + /// Example: Within the context of a function, a local definition. Could be a variable, or an inner function, type, + /// etc... + case local + + /// Example: An argument to a function, including generic parameters + case argument + + /// Example: Within the context of a class, struct, etc..., possibly nested in a function, references to definitions + /// at the container level. + case container + + /// Example: Within the context of a class, struct, etc..., possibly nested in a function, references to definitions + /// from an inherited class or protocol + case inheritedContainer + + /// Example: Referring to a type in an outer container, for example, within the iterator for a Sequence, referring to + /// Element + case outerContainer + + /// Example: Global variables, free functions, top level types + case global + + /// Example: Keywords + case inapplicable + + /// Provider doesn't know the relation between the completion and the context + case unknown + + /// Example: Provider was written before this enum existed, and didn't have an opportunity to provide a value + case unspecified +} + +extension ScopeProximity: BinaryCodable { + package init(_ decoder: inout BinaryDecoder) throws { + self = try decoder.decodeEnumByte { decoder, n in + switch n { + case 0: return .local + case 1: return .argument + case 2: return .container + case 3: return .inheritedContainer + case 4: return .outerContainer + case 5: return .global + case 6: return .inapplicable + case 7: return .unknown + case 8: return .unspecified + default: return nil + } + } + } + + package func encode(_ encoder: inout BinaryEncoder) { + let value: UInt8 + switch self { + case .local: value = 0 + case .argument: value = 1 + case .container: value = 2 + case .inheritedContainer: value = 3 + case .outerContainer: value = 4 + case .global: value = 5 + case .inapplicable: value = 6 + case .unknown: value = 7 + case .unspecified: value = 8 + } + encoder.write(value) + } +} diff --git a/Sources/CompletionScoring/Semantics/Components/StructuralProximity.swift b/Sources/CompletionScoring/Semantics/Components/StructuralProximity.swift new file mode 100644 index 000000000..7c56f6184 --- /dev/null +++ b/Sources/CompletionScoring/Semantics/Components/StructuralProximity.swift @@ -0,0 +1,58 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +package enum StructuralProximity: Equatable { + /// Example: Definition is in Project/Framework/UI/View.swift, usage site is Project/Framework/Model/View.swift, + /// so hops == 2, up one, and into a sibling. Hops is edit distance where the operations are 'delete, add', not replace. + case project(fileSystemHops: Int?) + + /// Example: Source of completion is from NSObject.h in the SDK + case sdk + + /// Example: Keyword + case inapplicable + + /// Example: Provider doesn't keep track of where definitions come from + case unknown + + /// Example: Provider was written before this enum existed, and didn't have an opportunity to provide a value + case unspecified +} + +extension StructuralProximity: BinaryCodable { + package init(_ decoder: inout BinaryDecoder) throws { + self = try decoder.decodeEnumByte { decoder, n in + switch n { + case 0: return .project(fileSystemHops: try Int?(&decoder)) + case 1: return .sdk + case 2: return .inapplicable + case 3: return .unknown + case 4: return .unspecified + default: return nil + } + } + } + + package func encode(_ encoder: inout BinaryEncoder) { + switch self { + case .project(let hops): + encoder.writeByte(0) + encoder.write(hops) + case .sdk: encoder.writeByte(1) + case .inapplicable: encoder.writeByte(2) + case .unknown: encoder.writeByte(3) + case .unspecified: encoder.writeByte(4) + } + } +} diff --git a/Sources/CompletionScoring/Semantics/Components/SynchronicityCompatibility.swift b/Sources/CompletionScoring/Semantics/Components/SynchronicityCompatibility.swift new file mode 100644 index 000000000..f51158f72 --- /dev/null +++ b/Sources/CompletionScoring/Semantics/Components/SynchronicityCompatibility.swift @@ -0,0 +1,62 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +package enum SynchronicityCompatibility: Equatable { + /// Example: sync->sync, async->sync, async->await async + case compatible + + /// Example: async->async without await + case convertible + + /// Example: sync->async + case incompatible + + /// Example: Accessing a type, using a keyword + case inapplicable + + /// Example: Not confident about either the context, or the target + case unknown + + /// Example: Provider was written before this enum existed, and didn't have an opportunity to provide a value + case unspecified +} + +extension SynchronicityCompatibility: BinaryCodable { + package init(_ decoder: inout BinaryDecoder) throws { + self = try decoder.decodeEnumByte { decoder, n in + switch n { + case 0: return .compatible + case 1: return .convertible + case 2: return .incompatible + case 3: return .inapplicable + case 4: return .unknown + case 5: return .unspecified + default: return nil + } + } + } + + package func encode(_ encoder: inout BinaryEncoder) { + let value: UInt8 + switch self { + case .compatible: value = 0 + case .convertible: value = 1 + case .incompatible: value = 2 + case .inapplicable: value = 3 + case .unknown: value = 4 + case .unspecified: value = 5 + } + encoder.write(value) + } +} diff --git a/Sources/CompletionScoring/Semantics/Components/TypeCompatibility.swift b/Sources/CompletionScoring/Semantics/Components/TypeCompatibility.swift new file mode 100644 index 000000000..3b69805e2 --- /dev/null +++ b/Sources/CompletionScoring/Semantics/Components/TypeCompatibility.swift @@ -0,0 +1,77 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +package enum TypeCompatibility: Equatable { + /// Examples: + /// - `String` is compatible with `String` + /// - `String` is compatible with `String?` + /// - `TextField` is compatible with `View`. + case compatible + + /// Example: `String` is unrelated to `Int` + case unrelated + + /// Example: `void` is invalid for `String` + case invalid + + /// Example: doesn't have a type: a keyword like 'try', or 'while' + case inapplicable + + /// Example: Failed to type check the expression context + case unknown + + /// Example: Provider was written before this enum existed, and didn't have an opportunity to provide a value + case unspecified +} + +extension TypeCompatibility { + /// This used to penalize producing an `Int` when `Int?` was expected, which isn't correct. + /// We no longer ask providers to make this distinction + @available(*, deprecated, renamed: "compatible") + package static let same = Self.compatible + + /// This used to penalize producing an `Int` when `Int?` was expected, which isn't correct. + /// We no longer ask providers to make this distinction + @available(*, deprecated, renamed: "compatible") + package static let implicitlyConvertible = Self.compatible +} + +extension TypeCompatibility: BinaryCodable { + package init(_ decoder: inout BinaryDecoder) throws { + self = try decoder.decodeEnumByte { decoder, n in + switch n { + case 0: return .compatible + case 1: return .unrelated + case 2: return .invalid + case 3: return .inapplicable + case 4: return .unknown + case 5: return .unspecified + default: return nil + } + } + } + + package func encode(_ encoder: inout BinaryEncoder) { + let value: UInt8 + switch self { + case .compatible: value = 0 + case .unrelated: value = 1 + case .invalid: value = 2 + case .inapplicable: value = 3 + case .unknown: value = 4 + case .unspecified: value = 5 + } + encoder.write(value) + } +} diff --git a/Sources/CompletionScoring/Semantics/SemanticClassification.swift b/Sources/CompletionScoring/Semantics/SemanticClassification.swift new file mode 100644 index 000000000..c4f13e9ec --- /dev/null +++ b/Sources/CompletionScoring/Semantics/SemanticClassification.swift @@ -0,0 +1,362 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +package struct SemanticClassification: Equatable { + package var completionKind: CompletionKind + package var popularity: Popularity + package var moduleProximity: ModuleProximity + package var scopeProximity: ScopeProximity + package var structuralProximity: StructuralProximity + package var typeCompatibility: TypeCompatibility + package var synchronicityCompatibility: SynchronicityCompatibility + package var availability: Availability + package var flair: Flair + + /// - Note: There is no natural order to these arguments, so they're alphabetical. + package init( + availability: Availability, + completionKind: CompletionKind, + flair: Flair, + moduleProximity: ModuleProximity, + popularity: Popularity, + scopeProximity: ScopeProximity, + structuralProximity: StructuralProximity, + synchronicityCompatibility: SynchronicityCompatibility, + typeCompatibility: TypeCompatibility + ) { + self.availability = availability + self.completionKind = completionKind + self.flair = flair + self.moduleProximity = moduleProximity + self.popularity = popularity + self.scopeProximity = scopeProximity + self.structuralProximity = structuralProximity + self.synchronicityCompatibility = synchronicityCompatibility + self.typeCompatibility = typeCompatibility + } + + package var score: Double { + let score = + availability.scoreComponent + * completionKind.scoreComponent + * flair.scoreComponent + * moduleProximity.scoreComponent + * popularity.scoreComponent + * scopeProximity.scoreComponent + * structuralProximity.scoreComponent + * synchronicityCompatibility.scoreComponent + * typeCompatibility.scoreComponent + * globalVariablesPenalty + + return score + } + + private var globalVariablesPenalty: Double { + // Global types and functions are fine, global variables and c enum cases in the global space are not. + if (scopeProximity == .global) && ((completionKind == .variable) || (completionKind == .enumCase)) { + return 0.75 + } + return 1.0 + } + + package struct ComponentDebugDescription { + package let name: String + package let instance: String + package let scoreComponent: Double + } + + private var scoreComponents: [CompletionScoreComponent] { + return [ + availability, + completionKind, + flair, + RawCompletionScoreComponent( + name: "symbolPopularity", + instance: "\(popularity.symbolComponent)", + scoreComponent: popularity.symbolComponent + ), + RawCompletionScoreComponent( + name: "modulePopularity", + instance: "\(popularity.moduleComponent)", + scoreComponent: popularity.moduleComponent + ), + moduleProximity, + scopeProximity, + structuralProximity, + synchronicityCompatibility, + typeCompatibility, + RawCompletionScoreComponent( + name: "globalVariablesPenalty", + instance: "\(globalVariablesPenalty != 1.0)", + scoreComponent: globalVariablesPenalty + ), + ] + } + + package var componentsDebugDescription: [ComponentDebugDescription] { + return scoreComponents.map { $0.componentDebugDescription } + } +} + +extension SemanticClassification: BinaryCodable { + package init(_ decoder: inout BinaryDecoder) throws { + availability = try Availability(&decoder) + completionKind = try CompletionKind(&decoder) + flair = try Flair(&decoder) + moduleProximity = try ModuleProximity(&decoder) + popularity = try Popularity(&decoder) + scopeProximity = try ScopeProximity(&decoder) + structuralProximity = try StructuralProximity(&decoder) + synchronicityCompatibility = try SynchronicityCompatibility(&decoder) + typeCompatibility = try TypeCompatibility(&decoder) + } + + package func encode(_ encoder: inout BinaryEncoder) { + encoder.write(availability) + encoder.write(completionKind) + encoder.write(flair) + encoder.write(moduleProximity) + encoder.write(popularity) + encoder.write(scopeProximity) + encoder.write(structuralProximity) + encoder.write(synchronicityCompatibility) + encoder.write(typeCompatibility) + } +} + +/// Published serialization methods +extension SemanticClassification { + package func byteRepresentation() -> [UInt8] { + binaryCodedRepresentation(contentVersion: 0) + } + + package init(byteRepresentation: [UInt8]) throws { + try self.init(binaryCodedRepresentation: byteRepresentation) + } + + package static func byteRepresentation(classifications: [Self]) -> [UInt8] { + classifications.binaryCodedRepresentation(contentVersion: 0) + } + + package static func classifications(byteRepresentations: [UInt8]) throws -> [Self] { + try [Self].init(binaryCodedRepresentation: byteRepresentations) + } +} + +/// Used for debugging. +fileprivate protocol CompletionScoreComponent { + var name: String { get } + + var instance: String { get } + + /// Return a value in 0...2. + /// + /// Think of values between 0 and 1 as penalties, 1 as neutral, and values from 1 and 2 as bonuses. + var scoreComponent: Double { get } +} + +extension CompletionScoreComponent { + var componentDebugDescription: SemanticClassification.ComponentDebugDescription { + return .init(name: name, instance: instance, scoreComponent: scoreComponent) + } +} + +/// Used for components that don't have a dedicated model. +private struct RawCompletionScoreComponent: CompletionScoreComponent { + let name: String + let instance: String + let scoreComponent: Double +} + +internal let unknownScore = 0.750 +internal let inapplicableScore = 1.000 +internal let unspecifiedScore = 1.000 + +private let localVariableScore = CompletionKind.variable.scoreComponent * ScopeProximity.local.scoreComponent +private let globalTypeScore = CompletionKind.type.scoreComponent * ScopeProximity.global.scoreComponent +internal let localVariableToGlobalTypeScoreRatio = localVariableScore / globalTypeScore + +extension CompletionKind: CompletionScoreComponent { + fileprivate var name: String { "CompletionKind" } + fileprivate var instance: String { "\(self)" } + fileprivate var scoreComponent: Double { + switch self { + case .keyword: return 1.000 + case .enumCase: return 1.100 + case .variable: return 1.075 + case .initializer: return 1.020 + case .argumentLabels: return 2.000 + case .function: return 1.025 + case .type: return 1.025 + case .template: return 1.100 + case .module: return 0.925 + case .other: return 1.000 + + case .unspecified: return unspecifiedScore + case .unknown: return unknownScore + } + } +} + +extension Flair: CompletionScoreComponent { + fileprivate var name: String { "Flair" } + fileprivate var instance: String { "\(self.debugDescription)" } + internal var scoreComponent: Double { + var total = 1.0 + if self.contains(.oldExpressionSpecific_pleaseAddSpecificCaseToThisEnum) { + total *= 1.5 + } + if self.contains(.chainedCallToSuper) { + total *= 1.5 + } + if self.contains(.chainedMember) { + total *= 0.3 + } + if self.contains(.swiftUIModifierOnSelfWhileBuildingSelf) { + total *= 0.3 + } + if self.contains(.swiftUIUnlikelyViewMember) { + total *= 0.125 + } + if self.contains(.commonKeywordAtCurrentPosition) { + total *= 1.25 + } + if self.contains(.rareKeywordAtCurrentPosition) { + total *= 0.75 + } + if self.contains(.rareTypeAtCurrentPosition) { + total *= 0.75 + } + if self.contains(.expressionAtNonScriptOrMainFileScope) { + total *= 0.125 + } + if self.contains(.rareMemberWithCommonName) { + total *= 0.75 + } + if self.contains(._situationallyLikely) { + total *= 1.25 + } + if self.contains(._situationallyUnlikely) { + total *= 0.75 + } + if self.contains(._situationallyInvalid) { + total *= 0.125 + } + return total + } +} + +extension ModuleProximity: CompletionScoreComponent { + fileprivate var name: String { "ModuleProximity" } + fileprivate var instance: String { "\(self)" } + fileprivate var scoreComponent: Double { + switch self { + case .imported(0): return 1.0500 + case .imported(1): return 1.0250 + case .imported(_): return 1.0125 + case .importable: return 0.5000 + case .invalid: return 0.2500 + + case .inapplicable: return inapplicableScore + case .unspecified: return unspecifiedScore + case .unknown: return unknownScore + } + } +} + +extension ScopeProximity: CompletionScoreComponent { + fileprivate var name: String { "ScopeProximity" } + fileprivate var instance: String { "\(self)" } + fileprivate var scoreComponent: Double { + switch self { + case .local: return 1.500 + case .argument: return 1.450 + case .container: return 1.350 + case .inheritedContainer: return 1.325 + case .outerContainer: return 1.325 + case .global: return 0.950 + + case .inapplicable: return inapplicableScore + case .unspecified: return unspecifiedScore + case .unknown: return unknownScore + } + } +} + +extension StructuralProximity: CompletionScoreComponent { + fileprivate var name: String { "StructuralProximity" } + fileprivate var instance: String { "\(self)" } + fileprivate var scoreComponent: Double { + switch self { + case .project(fileSystemHops: 0): return 1.010 + case .project(fileSystemHops: 1): return 1.005 + case .project(fileSystemHops: _): return 1.000 + case .sdk: return 0.995 + + case .inapplicable: return inapplicableScore + case .unspecified: return unspecifiedScore + case .unknown: return unknownScore + } + } +} + +extension SynchronicityCompatibility: CompletionScoreComponent { + fileprivate var name: String { "SynchronicityCompatibility" } + fileprivate var instance: String { "\(self)" } + fileprivate var scoreComponent: Double { + switch self { + case .compatible: return 1.00 + case .convertible: return 0.90 + case .incompatible: return 0.50 + + case .inapplicable: return inapplicableScore + case .unspecified: return unspecifiedScore + case .unknown: return unknownScore + } + } +} + +extension TypeCompatibility: CompletionScoreComponent { + fileprivate var name: String { "TypeCompatibility" } + fileprivate var instance: String { "\(self)" } + fileprivate var scoreComponent: Double { + switch self { + case .compatible: return 1.300 + case .unrelated: return 0.900 + case .invalid: return 0.300 + + case .inapplicable: return inapplicableScore + case .unspecified: return unspecifiedScore + case .unknown: return unknownScore + } + } +} + +extension Availability: CompletionScoreComponent { + fileprivate var name: String { "Availability" } + fileprivate var instance: String { "\(self)" } + internal var scoreComponent: Double { + switch self { + case .available: return 1.00 + case .unavailable: return 0.40 + case .softDeprecated, + .deprecated: + return 0.50 + + case .inapplicable: return inapplicableScore + case .unspecified: return unspecifiedScore + case .unknown: return unknownScore + } + } +} diff --git a/Sources/CompletionScoring/Text/CandidateBatch.swift b/Sources/CompletionScoring/Text/CandidateBatch.swift new file mode 100644 index 000000000..4e846d05b --- /dev/null +++ b/Sources/CompletionScoring/Text/CandidateBatch.swift @@ -0,0 +1,584 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +/// A list of possible code completion results that need to be culled and sorted based on a ``Pattern``. +package struct CandidateBatch: Sendable { + package typealias UTF8Bytes = Pattern.UTF8Bytes + package typealias ContentType = Candidate.ContentType + + /// Clients can access this via `CandidateBatch.withUnsafeStorage()` so that they can have read-access without the + /// overhead of runtime exclusitivy checks that happen when you read through a reference type. + package struct UnsafeStorage { + var bytes: UnsafeArray + var candidateByteOffsets: UnsafeArray + var filters: UnsafeArray + var contentTypes: UnsafeArray + + private init( + bytes: UnsafeArray, + candidateByteOffsets: UnsafeArray, + filters: UnsafeArray, + contentTypes: UnsafeArray + ) { + self.bytes = bytes + self.candidateByteOffsets = candidateByteOffsets + self.filters = filters + self.contentTypes = contentTypes + } + + static func allocate(candidateCapacity: Int, byteCapacity: Int) -> Self { + var candidateByteOffsets = UnsafeArray.allocate(initialCapacity: candidateCapacity + 1) + candidateByteOffsets.append(0) // Always contains the 'endIndex' + return Self( + bytes: UnsafeArray.allocate(initialCapacity: byteCapacity), + candidateByteOffsets: candidateByteOffsets, + filters: UnsafeArray.allocate(initialCapacity: candidateCapacity), + contentTypes: UnsafeArray.allocate(initialCapacity: candidateCapacity) + ) + } + + mutating func deallocate() { + bytes.deallocate() + candidateByteOffsets.deallocate() + filters.deallocate() + contentTypes.deallocate() + } + + func allocateCopy() -> Self { + return Self( + bytes: bytes.allocateCopy(preservingCapacity: true), + candidateByteOffsets: candidateByteOffsets.allocateCopy(preservingCapacity: true), + filters: filters.allocateCopy(preservingCapacity: true), + contentTypes: contentTypes.allocateCopy(preservingCapacity: true) + ) + } + + @inline(__always) + func bytes(at index: Int) -> UTF8Bytes { + let position = candidateByteOffsets[index] + let nextPosition = candidateByteOffsets[index + 1] + return UnsafeBufferPointer(start: bytes.elements.advanced(by: position), count: nextPosition - position) + } + + @inline(__always) + func candidateContent(at index: Int) -> (UTF8Bytes, ContentType) { + let position = candidateByteOffsets[index] + let nextPosition = candidateByteOffsets[index + 1] + let bytes = UnsafeBufferPointer( + start: bytes.elements.advanced(by: position), + count: nextPosition - position + ) + let contentType = contentTypes[index] + return (bytes, contentType) + } + + var count: Int { + filters.count + } + + package var indices: Range { + return 0.. Candidate { + Candidate(bytes: bytes(at: index), contentType: contentTypes[index], rejectionFilter: filters[index]) + } + + /// Don't add a method that returns a candidate, the candidates have unsafe pointers back into the batch, and + /// must not outlive it. + @inline(__always) + func enumerate(body: (Candidate) throws -> ()) rethrows { + for idx in 0.., body: (Int, Candidate) throws -> ()) rethrows { + precondition(range.lowerBound >= 0) + precondition(range.upperBound <= count) + for idx in range { + try body(idx, candidate(at: idx)) + } + } + + subscript(stringAt index: Int) -> String { + // Started as a valid string, so UTF8 must be valid, if this ever fails (would have to be something like a + // string with unvalidated content), we should fix it on the input side, not here. + return String(bytes: bytes(at: index), encoding: .utf8).unwrap(orFail: "Invalid UTF8 Sequence") + } + + mutating func append(_ candidate: String, contentType: ContentType) { + candidate.withUncachedUTF8Bytes { bytes in + append(candidateBytes: bytes, contentType: contentType, rejectionFilter: RejectionFilter(bytes: bytes)) + } + } + + mutating func append(_ candidate: Candidate) { + append( + candidateBytes: candidate.bytes, + contentType: candidate.contentType, + rejectionFilter: candidate.rejectionFilter + ) + } + + mutating func append(_ bytes: UTF8Bytes, contentType: ContentType) { + append(Candidate(bytes: bytes, contentType: contentType, rejectionFilter: .init(bytes: bytes))) + } + + mutating func append( + candidateBytes: some Collection, + contentType: ContentType, + rejectionFilter: RejectionFilter + ) { + bytes.append(contentsOf: candidateBytes) + filters.append(rejectionFilter) + contentTypes.append(contentType) + candidateByteOffsets.append(bytes.count) + } + + mutating func append(contentsOf candidates: [String], contentType: ContentType) { + filters.reserve(minimumAdditionalCapacity: candidates.count) + contentTypes.reserve(minimumAdditionalCapacity: candidates.count) + candidateByteOffsets.reserve(minimumAdditionalCapacity: candidates.count) + for text in candidates { + append(text, contentType: contentType) + } + } + + mutating func append(contentsOf candidates: [UTF8Bytes], contentType: ContentType) { + filters.reserve(minimumAdditionalCapacity: candidates.count) + contentTypes.reserve(minimumAdditionalCapacity: candidates.count) + candidateByteOffsets.reserve(minimumAdditionalCapacity: candidates.count) + for text in candidates { + append(text, contentType: contentType) + } + } + } + + private final class StorageBox { + /// Column oriented data for better cache performance. + var storage: UnsafeStorage + + private init(storage: UnsafeStorage) { + self.storage = storage + } + + init(candidateCapacity: Int, byteCapacity: Int) { + storage = UnsafeStorage.allocate(candidateCapacity: candidateCapacity, byteCapacity: byteCapacity) + } + + func copy() -> Self { + Self(storage: storage.allocateCopy()) + } + + deinit { + storage.deallocate() + } + } + + // `nonisolated(unsafe)` is fine because this `CandidateBatch` is the only struct with access to the `StorageBox`. + // All mutating access go through `mutate`, which copies `StorageBox` if `CandidateBatch` is not uniquely + // referenced. + nonisolated(unsafe) private var __storageBox_useAccessor: StorageBox + private var readonlyStorage: UnsafeStorage { + __storageBox_useAccessor.storage + } + + package init(byteCapacity: Int) { + self.init(candidateCapacity: byteCapacity / 16, byteCapacity: byteCapacity) + } + + private init(candidateCapacity: Int, byteCapacity: Int) { + __storageBox_useAccessor = StorageBox(candidateCapacity: candidateCapacity, byteCapacity: byteCapacity) + } + + package init() { + self.init(candidateCapacity: 0, byteCapacity: 0) + } + + package init(candidates: [String], contentType: ContentType) { + let byteCapacity = candidates.reduce(into: 0) { sum, string in + sum += string.utf8.count + } + self.init(candidateCapacity: candidates.count, byteCapacity: byteCapacity) + append(contentsOf: candidates, contentType: contentType) + } + + package init(candidates: [UTF8Bytes], contentType: ContentType) { + let byteCapacity = candidates.reduce(into: 0) { sum, candidate in + sum += candidate.count + } + self.init(candidateCapacity: candidates.count, byteCapacity: byteCapacity) + append(contentsOf: candidates, contentType: contentType) + } + + package func enumerate(body: (Candidate) throws -> ()) rethrows { + try readonlyStorage.enumerate(body: body) + } + + package func enumerate(body: (Int, Candidate) throws -> ()) rethrows { + try readonlyStorage.enumerate(0.., body: (Int, Candidate) throws -> ()) rethrows { + try readonlyStorage.enumerate(range, body: body) + } + + package func withAccessToCandidate(at idx: Int, body: (Candidate) throws -> R) rethrows -> R { + try withUnsafeStorage { storage in + try body(storage.candidate(at: idx)) + } + } + + package func withAccessToBytes(at idx: Int, body: (UTF8Bytes) throws -> R) rethrows -> R { + try withUnsafeStorage { storage in + try body(storage.bytes(at: idx)) + } + } + + package func withUnsafeStorage(_ body: (UnsafeStorage) throws -> R) rethrows -> R { + try withExtendedLifetime(__storageBox_useAccessor) { + try body(__storageBox_useAccessor.storage) + } + } + + static func withUnsafeStorages(_ batches: [Self], _ body: (UnsafeBufferPointer) -> R) -> R { + withExtendedLifetime(batches) { + withUnsafeTemporaryAllocation(of: UnsafeStorage.self, capacity: batches.count) { storages in + for (index, batch) in batches.enumerated() { + storages.initialize(index: index, to: batch.readonlyStorage) + } + let result = body(UnsafeBufferPointer(storages)) + storages.deinitializeAll() + return result + } + } + } + + package subscript(stringAt index: Int) -> String { + readonlyStorage[stringAt: index] + } + + package var count: Int { + return readonlyStorage.count + } + + package var indices: Range { + return readonlyStorage.indices + } + + var hasContent: Bool { + count > 0 + } + + private mutating func mutate(body: (inout UnsafeStorage) -> ()) { + if !isKnownUniquelyReferenced(&__storageBox_useAccessor) { + __storageBox_useAccessor = __storageBox_useAccessor.copy() + } + body(&__storageBox_useAccessor.storage) + } + + package mutating func append(_ candidate: String, contentType: ContentType) { + mutate { storage in + storage.append(candidate, contentType: contentType) + } + } + + package mutating func append(_ candidate: Candidate) { + mutate { storage in + storage.append(candidate) + } + } + + package mutating func append(_ candidate: UTF8Bytes, contentType: ContentType) { + mutate { storage in + storage.append(candidate, contentType: contentType) + } + } + + package mutating func append(contentsOf candidates: [String], contentType: ContentType) { + mutate { storage in + storage.append(contentsOf: candidates, contentType: contentType) + } + } + + package mutating func append(contentsOf candidates: [UTF8Bytes], contentType: ContentType) { + mutate { storage in + storage.append(contentsOf: candidates, contentType: contentType) + } + } + + package func filter(keepWhere predicate: (Int, Candidate) -> Bool) -> CandidateBatch { + var copy = CandidateBatch() + enumerate { index, candidate in + if predicate(index, candidate) { + copy.append(candidate) + } + } + return copy + } +} + +extension Pattern { + package struct CandidateBatchesMatch: Equatable, CustomStringConvertible { + package var batchIndex: Int + package var candidateIndex: Int + package var textScore: Double + + package init(batchIndex: Int, candidateIndex: Int, textScore: Double) { + self.batchIndex = batchIndex + self.candidateIndex = candidateIndex + self.textScore = textScore + } + + package var description: String { + "CandidateBatchesMatch(batch: \(batchIndex), candidateIndex: \(candidateIndex), score: \(textScore))" + } + } + + /// Represents work to be done by each thread when parallelizing scoring. The work is divided ahead of time to maximize memory locality. + /// Represents work to be done by each thread when parallelizing scoring. + /// The work is divided ahead of time to maximize memory locality. + /// + /// If we have 3 threads, A, B, and C, and 9 things to match, we want to assign the work like AAABBBCCC, not ABCABCABC, to maximize memory locality. + /// So if we had 2 candidate batches of 8 candidates each, and 3 threads, the new code divides them like [AAAAABBB][BBCCCCCC] + /// We expect parallelism in the ballpark of 4-64 threads, and the number of candidates to be in the 1,000-100,000 range. + /// So the remainder of having 1 thread process a few extra candidates doesn't matter. + /// + /// + /// Equatable for testing. + package struct ScoringWorkload: Equatable { + package struct CandidateBatchSlice: Equatable { + var batchIndex: Int + var candidateRange: Range + + package init(batchIndex: Int, candidateRange: Range) { + self.batchIndex = batchIndex + self.candidateRange = candidateRange + } + } + /// When scoring and matching and storing the results in a shared buffer, this is the base output index for this + /// thread workload. + var outputStartIndex: Int + var slices: [CandidateBatchSlice] = [] + + package init(outputStartIndex: Int, slices: [CandidateBatchSlice] = []) { + self.outputStartIndex = outputStartIndex + self.slices = slices + } + + package static func workloads( + for batches: [CandidateBatch], + parallelism threads: Int + ) + -> [ScoringWorkload] + { // Internal for testing. + let crossBatchCandidateCount = totalCandidates(batches: batches) + let budgetPerScoringWorkload = crossBatchCandidateCount / threads + let budgetPerScoringWorkloadRemainder = crossBatchCandidateCount - (budgetPerScoringWorkload * threads) + + var batchIndex = 0 + var candidateIndexInBatch = 0 + var workloads: [ScoringWorkload] = [] + var globalOutputIndex = 0 + for workloadIndex in 0.. Int { + batches.reduce(into: 0) { sum, batch in + sum += batch.count + } + } + + /// Find all of the matches across `batches` and score them, returning the scored results. + /// + /// This is a first part of selecting matches. Later the matches will be combined with matches from other providers, + /// where we'll pick the best matches and sort them with `selectBestMatches(from:textProvider:)` + package func scoredMatches(across batches: [CandidateBatch], precision: Precision) -> [CandidateBatchesMatch] { + compactScratchArea(capacity: Self.totalCandidates(batches: batches)) { matchesScratchArea in + let scoringWorkloads = ScoringWorkload.workloads( + for: batches, + parallelism: ProcessInfo.processInfo.processorCount + ) + // `nonisolated(unsafe)` is fine because every iteration accesses a distinct index of the buffer. + nonisolated(unsafe) let matchesScratchArea = matchesScratchArea + scoringWorkloads.concurrentForEach { threadWorkload in + UnsafeStackAllocator.withUnsafeStackAllocator { allocator in + var outputIndex = threadWorkload.outputStartIndex + for slice in threadWorkload.slices { + batches[slice.batchIndex].enumerate(slice.candidateRange) { candidateIndex, candidate in + assert(matchesScratchArea[outputIndex] == nil) + if let score = self.matchAndScore( + candidate: candidate, + precision: precision, + allocator: &allocator + ) { + matchesScratchArea[outputIndex] = CandidateBatchesMatch( + batchIndex: slice.batchIndex, + candidateIndex: candidateIndex, + textScore: score.value + ) + } + outputIndex += 1 + } + } + } + } + } + } + + package struct CandidateBatchMatch: Equatable { + package var candidateIndex: Int + package var textScore: Double + + package init(candidateIndex: Int, textScore: Double) { + self.candidateIndex = candidateIndex + self.textScore = textScore + } + } + + package func scoredMatches(in batch: CandidateBatch, precision: Precision) -> [CandidateBatchMatch] { + scoredMatches(across: [batch], precision: precision).map { multiMatch in + CandidateBatchMatch(candidateIndex: multiMatch.candidateIndex, textScore: multiMatch.textScore) + } + } +} + +/// A single potential code completion result that can be scored against a ``Pattern``. +package struct Candidate { + package enum ContentType: Equatable { + /// A symbol found by code completion. + case codeCompletionSymbol + /// The name of a file in the project. + case fileName + /// A symbol defined in the project, which can be found by eg. the workspace symbols request. + case projectSymbol + case unknown + } + + package let bytes: Pattern.UTF8Bytes + package let contentType: ContentType + let rejectionFilter: RejectionFilter + + package static func withAccessToCandidate( + for text: String, + contentType: ContentType, + body: (Candidate) throws -> R + ) + rethrows -> R + { + var text = text + return try text.withUTF8 { bytes in + return try body(.init(bytes: bytes, contentType: contentType, rejectionFilter: .init(bytes: bytes))) + } + } + + /// For debugging + internal var text: String { + String(bytes: bytes, encoding: .utf8).unwrap(orFail: "UTF8 was prevalidated.") + } +} + +// Creates a buffer of `capacity` elements of type `T?`, each initially set to nil. +/// +/// After running `initialize`, returns all elements that were set to non-`nil` values. +private func compactScratchArea(capacity: Int, initialize: (UnsafeMutablePointer) -> ()) -> [T] { + let scratchArea = UnsafeMutablePointer.allocate(capacity: capacity) + scratchArea.initialize(repeating: nil, count: capacity) + defer { + scratchArea.deinitialize(count: capacity) // Should be a no-op + scratchArea.deallocate() + } + initialize(scratchArea) + return UnsafeMutableBufferPointer(start: scratchArea, count: capacity).compacted() +} + +extension Candidate: CustomStringConvertible { + package var description: String { + return String(bytes: bytes, encoding: .utf8) ?? "(Invalid UTF8 Sequence)" + } +} + +extension Candidate { + @available(*, deprecated, message: "Pass an explicit content type") + package static func withAccessToCandidate(for text: String, body: (Candidate) throws -> R) rethrows -> R { + try withAccessToCandidate(for: text, contentType: .codeCompletionSymbol, body: body) + } +} + +extension CandidateBatch: Equatable { + package static func == (_ lhs: Self, _ rhs: Self) -> Bool { + (lhs.count == rhs.count) + && lhs.indices.allSatisfy { index in + lhs.withAccessToCandidate(at: index) { lhs in + rhs.withAccessToCandidate(at: index) { rhs in + return equateBytes(lhs.bytes, rhs.bytes) + && lhs.contentType == rhs.contentType + && lhs.rejectionFilter == rhs.rejectionFilter + } + } + } + } +} + +extension CandidateBatch { + @available(*, deprecated, message: "Pass an explicit content type") + package init(candidates: [String] = []) { + self.init(candidates: candidates, contentType: .codeCompletionSymbol) + } + + @available(*, deprecated, message: "Pass an explicit content type") + package mutating func append(_ candidate: String) { + append(candidate, contentType: .codeCompletionSymbol) + } + @available(*, deprecated, message: "Pass an explicit content type") + package mutating func append(_ candidate: UTF8Bytes) { + append(candidate, contentType: .codeCompletionSymbol) + } + @available(*, deprecated, message: "Pass an explicit content type") + package mutating func append(contentsOf candidates: [String]) { + append(contentsOf: candidates, contentType: .codeCompletionSymbol) + } +} diff --git a/Sources/CompletionScoring/Text/InfluencingIdentifiers.swift b/Sources/CompletionScoring/Text/InfluencingIdentifiers.swift new file mode 100644 index 000000000..42d4c0f67 --- /dev/null +++ b/Sources/CompletionScoring/Text/InfluencingIdentifiers.swift @@ -0,0 +1,169 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +fileprivate typealias UTF8Bytes = Pattern.UTF8Bytes + +package struct InfluencingIdentifiers: Sendable { + // `nonisolated(unsafe)` is fine because the underlying buffer is not modified until `deallocate` is called and the + // struct must not be used anymore after `deallocate` was called. + private nonisolated(unsafe) let identifiers: UnsafeBufferPointer + + private init(identifiers: UnsafeBufferPointer) { + self.identifiers = identifiers + } + + private static func allocate(copyingTokenizedIdentifiers possiblyEmptyTokenizedIdentifiers: [[String]]) -> Self { + let tokenizedIdentifiers = possiblyEmptyTokenizedIdentifiers.filter { possiblyEmptyTokenizedIdentifier in + possiblyEmptyTokenizedIdentifier.count > 0 + } + let allocatedIdentifiers: [Identifier] = tokenizedIdentifiers.enumerated().map { + identifierIndex, + tokenizedIdentifier in + // First is 1, last is 0.9375, scale is linear. Only a small preference for the first word. Right now when + // we have two words, it's for cases like an argument label and internal name predicting the argument type + // or a variable name and its type predicting it's value. This scoring shows a slight affinity for the name. + let scoreScale = + (identifierIndex == 0) + ? 1 : 1 - (0.0625 * (Double(identifierIndex) / Double(tokenizedIdentifiers.count - 1))) + return Identifier.allocate(copyingTokenizedIdentifier: tokenizedIdentifier, scoreScale: scoreScale) + } + return InfluencingIdentifiers(identifiers: UnsafeBufferPointer.allocate(copyOf: allocatedIdentifiers)) + } + + private func deallocate() { + for identifier in identifiers { + identifier.deallocate() + } + identifiers.deallocate() + } + + /// Invoke `body` with an instance of `InfluencingIdentifiers` that refers to memory only valid during the scope of `body`. + /// This pattern is used so that this code has no referencing counting overhead. Using types like Array to represent the + /// tokens during scoring results in referencing counting costing ~30% of the work. To avoid that, we use unsafe + /// buffer pointers, and then this method to constrain lifetimes. + /// - Parameter identifiers: The influencing identifiers in most to least influencing order. + package static func withUnsafeInfluencingTokenizedIdentifiers( + _ tokenizedIdentifiers: [[String]], + body: (Self) throws -> R + ) rethrows -> R { + let allocatedIdentifiers = allocate(copyingTokenizedIdentifiers: tokenizedIdentifiers); + defer { allocatedIdentifiers.deallocate() } + return try body(allocatedIdentifiers) + } + + var hasContent: Bool { + identifiers.hasContent + } + + private func match(token: Token, candidate: Candidate, candidateTokenization: Pattern.Tokenization) -> Bool { + candidateTokenization.anySatisfy { candidateTokenRange in + if token.bytes.count == candidateTokenRange.count { + let candidateToken = UnsafeBufferPointer(rebasing: candidate.bytes[candidateTokenRange]) + let leadingByteMatches = token.bytes[0].lowercasedUTF8Byte == candidateToken[0].lowercasedUTF8Byte + return leadingByteMatches && equateBytes(token.bytes.afterFirst(), candidateToken.afterFirst()) + } + return false + } + } + + /// Returns a value between 0...1, where 0 indicates `Candidate` was not textually related to the identifiers, and 1.0 + /// indicates the candidate was strongly related to the identifiers. + /// + /// Currently, this is implemented by tokenizing the candidate and the identifiers, and then seeing if any of the tokens + /// match. If each identifier has one or more tokens in the candidate, return 1.0. If no tokens from the identifiers appear + /// in the candidate, return 0.0. + package func score(candidate: Candidate, allocator: inout UnsafeStackAllocator) -> Double { + var candidateTokenization: Pattern.Tokenization? = nil; + defer { candidateTokenization?.deallocate(allocator: &allocator) } + var score = 0.0 + for identifier in identifiers { + // TODO: We could turn this loop inside out to walk the candidate tokens first, and skip the ones that are shorter + // than the shortest token, or keep bit for each length we have, and skip almost all of them. + let matchedTokenCount = identifier.tokens.countOf { token in + if (RejectionFilter.match(pattern: token.rejectionFilter, candidate: candidate.rejectionFilter) + == .maybe) + { + let candidateTokenization = candidateTokenization.lazyInitialize { + Pattern.Tokenization.allocate( + mixedcaseBytes: candidate.bytes, + contentType: candidate.contentType, + allocator: &allocator + ) + } + return match(token: token, candidate: candidate, candidateTokenization: candidateTokenization) + } + return false + } + score = max(score, identifier.score(matchedTokenCount: matchedTokenCount)) + } + return score + } +} + +fileprivate extension InfluencingIdentifiers { + struct Identifier { + let tokens: UnsafeBufferPointer + private let scoreScale: Double + + private init(tokens: UnsafeBufferPointer, scoreScale: Double) { + self.tokens = tokens + self.scoreScale = scoreScale + } + + func deallocate() { + for token in tokens { + token.deallocate() + } + tokens.deallocate() + } + + static func allocate(copyingTokenizedIdentifier tokenizedIdentifier: [String], scoreScale: Double) -> Self { + return Identifier( + tokens: UnsafeBufferPointer.allocate(copyOf: tokenizedIdentifier.map(Token.allocate)), + scoreScale: scoreScale + ) + } + + /// Returns a value between 0...1 + func score(matchedTokenCount: Int) -> Double { + if matchedTokenCount == 0 { + return 0 + } else if tokens.count == 1 { // We matched them all, make it obvious we won't divide by 0. + return 1 * scoreScale + } else { + let p = Double(matchedTokenCount - 1) / Double(tokens.count - 1) + return (0.75 + (p * 0.25)) * scoreScale + } + } + } +} + +fileprivate extension InfluencingIdentifiers { + struct Token { + let bytes: UTF8Bytes + let rejectionFilter: RejectionFilter + private init(bytes: UTF8Bytes) { + self.bytes = bytes + self.rejectionFilter = RejectionFilter(bytes: bytes) + } + + static func allocate(_ text: String) -> Self { + Token(bytes: UnsafeBufferPointer.allocate(copyOf: text.utf8)) + } + + func deallocate() { + bytes.deallocate() + } + } +} diff --git a/Sources/CompletionScoring/Text/MatchCollator.Match.swift b/Sources/CompletionScoring/Text/MatchCollator.Match.swift new file mode 100644 index 000000000..c75b2a8ef --- /dev/null +++ b/Sources/CompletionScoring/Text/MatchCollator.Match.swift @@ -0,0 +1,48 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +extension MatchCollator { + package struct Match { + /// For client use, has no meaning to CompletionScoring, useful when mapping Match instances back to + /// higher level constructs. + package var identifier: Int + + package var batchIndex: Int + + package var candidateIndex: Int + + /// Items with the same (groupID, batchID) sort together. Initially used to locate types with their initializers. + package var groupID: Int? + + package var score: CompletionScore + + package init(batchIndex: Int, candidateIndex: Int, groupID: Int?, score: CompletionScore) { + self.init( + identifier: 0, + batchIndex: batchIndex, + candidateIndex: candidateIndex, + groupID: groupID, + score: score + ) + } + + package init(identifier: Int, batchIndex: Int, candidateIndex: Int, groupID: Int?, score: CompletionScore) { + self.identifier = identifier + self.batchIndex = batchIndex + self.candidateIndex = candidateIndex + self.groupID = groupID + self.score = score + } + } +} diff --git a/Sources/CompletionScoring/Text/MatchCollator.Selection.swift b/Sources/CompletionScoring/Text/MatchCollator.Selection.swift new file mode 100644 index 000000000..5b0c64370 --- /dev/null +++ b/Sources/CompletionScoring/Text/MatchCollator.Selection.swift @@ -0,0 +1,22 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +extension MatchCollator { + /// The result of best match selection. + package struct Selection { + /// The precision used during matching, which varies based on the number of candidates and the input pattern length. + package var precision: Pattern.Precision + package var matches: [Match] + } +} diff --git a/Sources/CompletionScoring/Text/MatchCollator.swift b/Sources/CompletionScoring/Text/MatchCollator.swift new file mode 100644 index 000000000..acec2c198 --- /dev/null +++ b/Sources/CompletionScoring/Text/MatchCollator.swift @@ -0,0 +1,546 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +/// Aggregates functionality to support the `MatchCollator.selectBestMatches(for:from:in:)` function, which sorts and +/// selects the best matches from a list, applying the `.thorough` scoring function while being conscience of it's expense. +package struct MatchCollator { + private var originalMatches: UnsafeBufferPointer + private var rescoredMatches: UnsafeMutableBufferPointer + private let batches: UnsafeBufferPointer + private let groupScores: UnsafeMutableBufferPointer + private let influencers: InfluencingIdentifiers + private let patternUTF8Length: Int + private let tieBreaker: UnsafePointer + private let maximumNumberOfItemsForExpensiveSelection: Int + package static let defaultMaximumNumberOfItemsForExpensiveSelection = 100 + + private init( + originalMatches: UnsafeBufferPointer, + rescoredMatches: UnsafeMutableBufferPointer, + batches: UnsafeBufferPointer, + groupScores: UnsafeMutableBufferPointer, + influencers: InfluencingIdentifiers, + patternUTF8Length: Int, + orderingTiesBy tieBreaker: UnsafePointer, + maximumNumberOfItemsForExpensiveSelection: Int + ) { + for match in originalMatches { + precondition(batches.indices.contains(match.batchIndex)) + precondition(batches[match.batchIndex].indices.contains(match.candidateIndex)) + } + self.originalMatches = originalMatches + self.rescoredMatches = rescoredMatches + self.batches = batches + self.groupScores = groupScores + self.influencers = influencers + self.patternUTF8Length = patternUTF8Length + self.tieBreaker = tieBreaker + self.maximumNumberOfItemsForExpensiveSelection = maximumNumberOfItemsForExpensiveSelection + } + + private static func withUnsafeMatchCollator( + matches originalMatches: [Match], + batches: [CandidateBatch], + influencingTokenizedIdentifiers: [[String]], + patternUTF8Length: Int, + orderingTiesBy tieBreakerBody: (_ lhs: Match, _ rhs: Match) -> Bool, + maximumNumberOfItemsForExpensiveSelection: Int, + body: (inout MatchCollator) -> R + ) -> R { + let rescoredMatches = UnsafeMutableBufferPointer.allocate(capacity: originalMatches.count); + defer { rescoredMatches.deinitializeAllAndDeallocate() } + let groupScores = UnsafeMutableBufferPointer.allocate(capacity: originalMatches.count); + defer { groupScores.deinitializeAllAndDeallocate() } + for (matchIndex, originalMatch) in originalMatches.enumerated() { + rescoredMatches.initialize( + index: matchIndex, + to: RescoredMatch( + originalMatchIndex: matchIndex, + textIndex: TextIndex(batch: originalMatch.batchIndex, candidate: originalMatch.candidateIndex), + denseGroupID: nil, + individualScore: originalMatch.score, + groupScore: -Double.infinity, + falseStarts: 0 + ) + ) + } + assignDenseGroupId(to: rescoredMatches, from: originalMatches, batchCount: batches.count) + return withoutActuallyEscaping(tieBreakerBody) { tieBreakerBody in + var tieBreaker = TieBreaker(tieBreakerBody) + return withExtendedLifetime(tieBreaker) { + InfluencingIdentifiers.withUnsafeInfluencingTokenizedIdentifiers(influencingTokenizedIdentifiers) { + influencers in + originalMatches.withUnsafeBufferPointer { originalMatches in + CandidateBatch.withUnsafeStorages(batches) { batchStorages in + var collator = Self( + originalMatches: originalMatches, + rescoredMatches: rescoredMatches, + batches: batchStorages, + groupScores: groupScores, + influencers: influencers, + patternUTF8Length: patternUTF8Length, + orderingTiesBy: &tieBreaker, + maximumNumberOfItemsForExpensiveSelection: maximumNumberOfItemsForExpensiveSelection + ) + return body(&collator) + } + } + } + } + } + } + + /// This allows us to only take the dictionary hit one time, so that we don't have to do repeated dictionary lookups + /// as we lookup groupIDs and map them to group scores. + private static func assignDenseGroupId( + to rescoredMatches: UnsafeMutableBufferPointer, + from originalMatches: [Match], + batchCount: Int + ) { + typealias SparseGroupID = Int + typealias DenseGroupID = Int + let initialDictionaryCapacity = (batchCount > 0) ? originalMatches.count / batchCount : 0 + var batchAssignments: [[SparseGroupID: DenseGroupID]] = Array( + repeating: Dictionary(capacity: initialDictionaryCapacity), + count: batchCount + ) + var nextDenseID = 0 + for (matchIndex, match) in originalMatches.enumerated() { + if let sparseID = match.groupID { + rescoredMatches[matchIndex].denseGroupID = batchAssignments[match.batchIndex][sparseID].lazyInitialize { + let denseID = nextDenseID + nextDenseID += 1 + return denseID + } + } + } + } + + private mutating func selectBestFastScoredMatchesForThoroughScoring() { + if rescoredMatches.count > maximumNumberOfItemsForExpensiveSelection { + rescoredMatches.selectTopKAndTruncate(maximumNumberOfItemsForExpensiveSelection) { lhs, rhs in + (lhs.groupScore >? rhs.groupScore) ?? (lhs.individualScore > rhs.individualScore) + } + } + } + + private func refreshGroupScores() { + // We call this the first time without initializing the Double values. + groupScores.setAll(to: -.infinity) + for match in rescoredMatches { + if let denseGroupID = match.denseGroupID { + groupScores[denseGroupID] = max(groupScores[denseGroupID], match.individualScore.value) + } + } + for (index, match) in rescoredMatches.enumerated() { + if let denseGroupID = match.denseGroupID { + rescoredMatches[index].groupScore = groupScores[denseGroupID] + } else { + rescoredMatches[index].groupScore = rescoredMatches[index].individualScore.value + } + } + } + + private func unsafeBytes(at textIndex: TextIndex) -> CandidateBatch.UTF8Bytes { + return batches[textIndex.batch].bytes(at: textIndex.candidate) + } + + mutating func thoroughlyRescore(pattern: Pattern) { + // `nonisolated(unsafe)` is fine because every iteration accesses a different index of `batches`. + nonisolated(unsafe) let batches = batches + // `nonisolated(unsafe)` is fine because every iteration accesses a disjunct set of indices of `rescoredMatches`. + nonisolated(unsafe) let rescoredMatches = rescoredMatches + let pattern = pattern + rescoredMatches.slicedConcurrentForEachSliceRange { sliceRange in + UnsafeStackAllocator.withUnsafeStackAllocator { allocator in + for matchIndex in sliceRange { + let textIndex = rescoredMatches[matchIndex].textIndex + let (candidateBytes, candidateContentType) = batches[textIndex.batch] + .candidateContent(at: textIndex.candidate) + let textScore = pattern.score( + candidate: candidateBytes, + contentType: candidateContentType, + precision: .thorough, + allocator: &allocator + ) + rescoredMatches[matchIndex].individualScore.textComponent = textScore.value + rescoredMatches[matchIndex].falseStarts = textScore.falseStarts + } + } + } + } + + /// Generated and validated by `MatchCollatorTests.testMinimumTextCutoff()` + package static let bestRejectedTextScoreByPatternLength: [Double] = [ + 0.0, + 0.0, + 2.900400881379344, + 2.900400881379344, + 2.900400881379344, + 2.900400881379344, + 2.900400881379344, + 2.900400881379344, + 2.900400881379344, + 2.900400881379344, + 2.900400881379344, + ] + + private var cutoffRatio: Double { + // | + // 0.67 | ____________________ + // | / + // |/ + // +------------------------ + // 4 + let fullCutoffRatio = (2.0 / 3.0) + let weight = min(max(Double(patternUTF8Length), 1.0) / 4.0, 1.0) + return fullCutoffRatio * weight + } + + private static let maxInfluenceBonus = 0.10 + + private static let maxFalseStarts = 2 + + private var bestRejectedTextScore: Double { + let bestRejectedTextScoreByPatternLength = Self.bestRejectedTextScoreByPatternLength + let inBounds = bestRejectedTextScoreByPatternLength.indices.contains(patternUTF8Length) + return + (inBounds + ? bestRejectedTextScoreByPatternLength[patternUTF8Length] : bestRejectedTextScoreByPatternLength.last) + ?? 0 + } + + private mutating func selectBestThoroughlyScoredMatches() { + if let bestThoroughlyScoredMatch = rescoredMatches.max(by: \.individualScore) { + let topMatchFalseStarts = bestThoroughlyScoredMatch.falseStarts + let compositeCutoff = self.cutoffRatio * bestThoroughlyScoredMatch.individualScore.value + let semanticCutoffForTokenFalseStartsExemption = + bestThoroughlyScoredMatch.individualScore.semanticComponent / 3.0 + let bestRejectedTextScore = bestRejectedTextScore + let maxAllowedFalseStarts = Self.maxFalseStarts + rescoredMatches.removeAndTruncateWhere { candidate in + let overcomesTextCutoff = candidate.individualScore.textComponent > bestRejectedTextScore + let overcomesFalseStartCutoff = candidate.falseStarts <= maxAllowedFalseStarts + let acceptedByCompositeScore = candidate.individualScore.value >= compositeCutoff + let acceptedByTokenFalseStarts = + (candidate.falseStarts <= topMatchFalseStarts) + && (candidate.individualScore.semanticComponent >= semanticCutoffForTokenFalseStartsExemption) + let keep = + overcomesTextCutoff && overcomesFalseStartCutoff + && (acceptedByCompositeScore || acceptedByTokenFalseStarts) + return !keep + } + } + } + + private mutating func selectBestFastScoredMatches() { + if let bestSemanticScore = rescoredMatches.max(of: { candidate in candidate.individualScore.semanticComponent }) { + let minimumSemanticScore = bestSemanticScore * cutoffRatio + rescoredMatches.removeAndTruncateWhere { candidate in + candidate.individualScore.semanticComponent < minimumSemanticScore + } + } + } + + private mutating func applyInfluence() { + let influencers = self.influencers + let maxInfluenceBonus = Self.maxInfluenceBonus + if influencers.hasContent && (maxInfluenceBonus != 0.0) { + // `nonisolated(unsafe)` is fine because every iteration accesses a disjoint set of indices in `rescoredMatches`. + nonisolated(unsafe) let rescoredMatches = rescoredMatches + // `nonisolated(unsafe)` is fine because `batches` is not modified + nonisolated(unsafe) let batches = batches + rescoredMatches.slicedConcurrentForEachSliceRange { sliceRange in + UnsafeStackAllocator.withUnsafeStackAllocator { allocator in + for matchIndex in sliceRange { + let textIndex = rescoredMatches[matchIndex].textIndex + let candidate = batches[textIndex.batch].candidate(at: textIndex.candidate) + let percentOfInfluenceBonus = influencers.score(candidate: candidate, allocator: &allocator) + let textCoefficient = (percentOfInfluenceBonus * maxInfluenceBonus) + 1.0 + rescoredMatches[matchIndex].individualScore.textComponent *= textCoefficient + } + } + } + refreshGroupScores() + } + } + + private func lessThan(_ lhs: RescoredMatch, _ rhs: RescoredMatch) -> Bool { + if let definitiveGroupScoreComparison = lhs.groupScore >? rhs.groupScore { + return definitiveGroupScoreComparison + // Only compare `individualScore` within the same group, or among items that have no group. + // Otherwise when the group score ties, we would interleave the members of the tying groups. + } else if (lhs.denseGroupID == rhs.denseGroupID), + let definitiveIndividualScoreComparison = lhs.individualScore.value >? rhs.individualScore.value + { + return definitiveIndividualScoreComparison + } else { + let lhsBytes = unsafeBytes(at: lhs.textIndex) + let rhsBytes = unsafeBytes(at: rhs.textIndex) + switch compareBytes(lhsBytes, rhsBytes) { + case .ascending: + return true + case .descending: + return false + case .same: + if rescoredMatches.count <= maximumNumberOfItemsForExpensiveSelection { + let lhsOriginal = originalMatches[lhs.originalMatchIndex] + let rhsOriginal = originalMatches[rhs.originalMatchIndex] + if tieBreaker.pointee.lessThan(lhsOriginal, rhsOriginal) { + return true + } else if tieBreaker.pointee.lessThan(rhsOriginal, lhsOriginal) { + return false + } + } + return (lhs.originalMatchIndex < rhs.originalMatchIndex) + } + } + } + + private mutating func sort() { + rescoredMatches.sort(by: lessThan) + } + + private mutating func selectBestMatches(pattern: Pattern) -> Selection { + refreshGroupScores() + let precision: Pattern.Precision + if pattern.typedEnoughForThoroughScoring + || (rescoredMatches.count <= maximumNumberOfItemsForExpensiveSelection) + { + selectBestFastScoredMatchesForThoroughScoring() + thoroughlyRescore(pattern: pattern) + refreshGroupScores() + selectBestThoroughlyScoredMatches() + precision = .thorough + } else { + selectBestFastScoredMatches() + precision = .fast + } + applyInfluence() + sort() + return Selection( + precision: precision, + matches: rescoredMatches.map { match in + originalMatches[match.originalMatchIndex] + } + ) + } + + /// Uses heuristics to cull matches, and then apply the expensive `.thorough` scoring function. + /// + /// Returns the results stably ordered by score, then text. + package static func selectBestMatches( + _ matches: [Match], + from batches: [CandidateBatch], + for pattern: Pattern, + influencingTokenizedIdentifiers: [[String]], + orderingTiesBy tieBreaker: (_ lhs: Match, _ rhs: Match) -> Bool, + maximumNumberOfItemsForExpensiveSelection: Int + ) -> Selection { + withUnsafeMatchCollator( + matches: matches, + batches: batches, + influencingTokenizedIdentifiers: influencingTokenizedIdentifiers, + patternUTF8Length: pattern.patternUTF8Length, + orderingTiesBy: tieBreaker, + maximumNumberOfItemsForExpensiveSelection: maximumNumberOfItemsForExpensiveSelection + ) { collator in + collator.selectBestMatches(pattern: pattern) + } + } + + /// Short for `selectBestMatches(_:from:for:influencingTokenizedIdentifiers:orderingTiesBy:).matches` + package static func selectBestMatches( + for pattern: Pattern, + from matches: [Match], + in batches: [CandidateBatch], + influencingTokenizedIdentifiers: [[String]], + orderingTiesBy tieBreaker: (_ lhs: Match, _ rhs: Match) -> Bool, + maximumNumberOfItemsForExpensiveSelection: Int = Self.defaultMaximumNumberOfItemsForExpensiveSelection + ) -> [Match] { + return selectBestMatches( + matches, + from: batches, + for: pattern, + influencingTokenizedIdentifiers: influencingTokenizedIdentifiers, + orderingTiesBy: tieBreaker, + maximumNumberOfItemsForExpensiveSelection: maximumNumberOfItemsForExpensiveSelection + ).matches + } + + /// Split identifiers into constituent subwords. For example "documentDownload" becomes ["document", "Download"] + /// - Parameters: + /// - identifiers: Strings from the program source, like "documentDownload" + /// - filterLowSignalTokens: When true, removes common tokens that would falsely signal influence, like "from". + /// - Returns: A value suitable for use with the `influencingTokenizedIdentifiers:` parameter of `selectBestMatches(…)`. + package static func tokenize( + influencingTokenizedIdentifiers identifiers: [String], + filterLowSignalTokens: Bool + ) -> [[String]] { + identifiers.map { identifier in + tokenize(influencingTokenizedIdentifier: identifier, filterLowSignalTokens: filterLowSignalTokens) + } + } + + /// Only package so that we can performance test this + package static func performanceTest_influenceScores( + for batches: [CandidateBatch], + influencingTokenizedIdentifiers: [[String]], + iterations: Int + ) -> Double { + let matches = batches.enumerated().flatMap { batchIndex, batch in + (0.. [String] + { + var tokens: [String] = [] + identifier.withUncachedUTF8Bytes { identifierBytes in + UnsafeStackAllocator.withUnsafeStackAllocator { allocator in + var tokenization = Pattern.Tokenization.allocate( + mixedcaseBytes: identifierBytes, + contentType: .codeCompletionSymbol, + allocator: &allocator + ); defer { tokenization.deallocate(allocator: &allocator) } + tokenization.enumerate { tokenRange in + if let token = String(bytes: identifierBytes[tokenRange], encoding: .utf8) { + tokens.append(token) + } + } + } + } + if filterLowSignalTokens { + let minimumLength = 4 // Shorter tokens appear too much to be useful: (in, on, a, the…) + let ignoredTokens: Set = ["from", "with"] + tokens.removeAll { token in + return (token.count < minimumLength) || ignoredTokens.contains(token.lowercased()) + } + } + return tokens + } +} + +extension MatchCollator { + fileprivate struct TextIndex { + var batch: Int + var candidate: Int + } +} + +extension MatchCollator { + fileprivate struct RescoredMatch { + var originalMatchIndex: Int + var textIndex: TextIndex + var denseGroupID: Int? + var individualScore: CompletionScore + var groupScore: Double + var falseStarts: Int + } +} + +extension MatchCollator { + /// A wrapper to allow taking an unsafe pointer to a closure. + fileprivate final class TieBreaker { + var lessThan: (_ lhs: Match, _ rhs: Match) -> Bool + + init(_ lessThan: @escaping (_ lhs: Match, _ rhs: Match) -> Bool) { + self.lessThan = lessThan + } + } +} + +extension MatchCollator.RescoredMatch: CustomStringConvertible { + var description: String { + func format(_ value: Double) -> String { + String(format: "%0.3f", value) + } + return + """ + RescoredMatch(\ + idx: \(originalMatchIndex), \ + gid: \(denseGroupID?.description ?? "_"), \ + score.t: \(format(individualScore.textComponent)), \ + score.s: \(format(individualScore.semanticComponent)), \ + groupScore: \(format(groupScore)), \ + falseStarts: \(falseStarts)\ + ) + """ + } +} + +fileprivate extension Pattern { + var typedEnoughForThoroughScoring: Bool { + patternUTF8Length >= MatchCollator.minimumPatternLengthToAlwaysRescoreWithThoroughPrecision + } +} + +// Deprecated Entry Points +extension MatchCollator { + @available(*, deprecated, renamed: "selectBestMatches(for:from:in:influencingTokenizedIdentifiers:orderingTiesBy:)") + package static func selectBestMatches( + for pattern: Pattern, + from matches: [Match], + in batches: [CandidateBatch], + influencingTokenizedIdentifiers: [[String]] + ) -> [Match] { + selectBestMatches( + for: pattern, + from: matches, + in: batches, + influencingTokenizedIdentifiers: influencingTokenizedIdentifiers, + orderingTiesBy: { _, _ in false } + ) + } + + @available( + *, + deprecated, + message: + "Use the MatchCollator.Selection.precision value returned from selectBestMatches(...) to choose between fast and thorough matched text ranges." + ) + package static var bestMatchesThoroughScanningMinimumPatternLength: Int { + minimumPatternLengthToAlwaysRescoreWithThoroughPrecision + } +} + +extension MatchCollator { + package static let minimumPatternLengthToAlwaysRescoreWithThoroughPrecision = 2 +} diff --git a/Sources/CompletionScoring/Text/Pattern.swift b/Sources/CompletionScoring/Text/Pattern.swift new file mode 100644 index 000000000..c0c3df73a --- /dev/null +++ b/Sources/CompletionScoring/Text/Pattern.swift @@ -0,0 +1,1276 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +/// The pattern that the user has typed and which should be used to cull and score results. +package final class Pattern: Sendable { + package typealias ContentType = Candidate.ContentType + package typealias UTF8Bytes = UnsafeBufferPointer + package typealias UTF8ByteRange = Range + package enum Precision { + case fast + case thorough + } + + package let text: String + + /// These all have pattern in the name to avoid confusion with their candidate counterparts in functions operating on both. + // `nonisolated(unsafe)` is fine because the underlying buffer is never modified. + private nonisolated(unsafe) let patternMixedcaseBytes: UTF8Bytes + private nonisolated(unsafe) let patternLowercaseBytes: UTF8Bytes + private let patternRejectionFilter: RejectionFilter + /// For each byte in `text`, a rejection filter that contains all the characters occurring after or at that offset. + /// + /// This way when we have already matched the first 4 bytes, we can check which characters occur from byte 5 onwards + /// and check that they appear in the candidate's remaining text. + private let patternSuccessiveRejectionFilters: [RejectionFilter] + private let patternHasMixedcase: Bool + internal var patternUTF8Length: Int { patternMixedcaseBytes.count } + + package init(text: String) { + self.text = text + let mixedcaseBytes = Array(text.utf8) + let lowercaseBytes = mixedcaseBytes.map(\.lowercasedUTF8Byte) + self.patternMixedcaseBytes = UnsafeBufferPointer.allocate(copyOf: mixedcaseBytes) + self.patternLowercaseBytes = UnsafeBufferPointer.allocate(copyOf: lowercaseBytes) + self.patternRejectionFilter = .init(lowercaseBytes: lowercaseBytes) + self.patternHasMixedcase = lowercaseBytes != mixedcaseBytes + self.patternSuccessiveRejectionFilters = UnsafeStackAllocator.withUnsafeStackAllocator { allocator in + return lowercaseBytes.withUnsafeBufferPointer { lowercaseBytes in + var rejectionFilters = Self.allocateSuccessiveRejectionFilters( + lowercaseBytes: lowercaseBytes, + allocator: &allocator + ) + defer { allocator.deallocate(&rejectionFilters) } + return Array(rejectionFilters) + } + } + } + + deinit { + patternMixedcaseBytes.deallocate() + patternLowercaseBytes.deallocate() + } + + /// Perform an insensitive greedy match and return the location where the greedy match stared if it succeeded. + /// If the match was not successful, return `nil`. + /// + /// Future searches used during scoring can use this location to jump past all of the initial bytes that don't match + /// anything. An empty pattern matches with location 0. + private func matchLocation(candidate: Candidate) -> Int? { + if RejectionFilter.match(pattern: patternRejectionFilter, candidate: candidate.rejectionFilter) == .maybe { + let cBytes = candidate.bytes + let pCount = patternLowercaseBytes.count + let cCount = cBytes.count + var pRemaining = pCount + var cRemaining = cCount + if (pRemaining > 0) && (cRemaining > 0) && (pRemaining <= cRemaining) { + let pFirst = patternLowercaseBytes[0] + let cStart = cBytes.firstIndex { cByte in + cByte.lowercasedUTF8Byte == pFirst + } + if let cStart = cStart { + cRemaining -= cStart + cRemaining -= 1 + pRemaining -= 1 + // While we're not at the end and the remaining pattern is as short or shorter than the remaining candidate. + while (pRemaining > 0) && (cRemaining > 0) && (pRemaining <= cRemaining) { + let cIdx = cCount - cRemaining + let pIdx = pCount - pRemaining + if cBytes[cIdx].lowercasedUTF8Byte == patternLowercaseBytes[pIdx] { + pRemaining -= 1 + } + cRemaining -= 1 + } + return (pRemaining == 0) ? cStart : nil + } + } + return pCount == 0 ? 0 : nil + } + return nil + } + + package func matches(candidate: Candidate) -> Bool { + matchLocation(candidate: candidate) != nil + } + + package func score(candidate: Candidate, precision: Precision) -> Double { + score(candidate: candidate.bytes, contentType: candidate.contentType, precision: precision) + } + + package func score( + candidate candidateMixedcaseBytes: UTF8Bytes, + contentType: ContentType, + precision: Precision + ) -> Double { + UnsafeStackAllocator.withUnsafeStackAllocator { allocator in + score( + candidate: candidateMixedcaseBytes, + contentType: contentType, + precision: precision, + allocator: &allocator + ) + .value + } + } + + package struct TextScore: Comparable { + package var value: Double + package var falseStarts: Int + + static let worstPossibleScore = TextScore(value: -Double.infinity, falseStarts: Int.max) + + package init(value: Double, falseStarts: Int) { + self.value = value + self.falseStarts = falseStarts + } + + package static func < (_ lhs: Self, _ rhs: Self) -> Bool { + return (lhs.value rhs.falseStarts) + } + + fileprivate static let emptyMatchScore = TextScore(value: 1.0, falseStarts: 0) + fileprivate static let noMatchScore = TextScore(value: 0.0, falseStarts: 0) + } + + internal func matchAndScore( + candidate: Candidate, + precision: Precision, + allocator: inout UnsafeStackAllocator + ) -> TextScore? { + return matchLocation(candidate: candidate).map { firstMatchingLowercaseByteIndex in + if patternLowercaseBytes.hasContent { + var indexedCandidate = IndexedCandidate.allocate( + referencing: candidate.bytes, + patternByteCount: patternLowercaseBytes.count, + firstMatchingLowercaseByteIndex: firstMatchingLowercaseByteIndex, + contentType: candidate.contentType, + allocator: &allocator + ) + defer { indexedCandidate.deallocate(allocator: &allocator) } + return score( + candidate: &indexedCandidate, + precision: precision, + captureMatchingRanges: false, + allocator: &allocator + ) + } else { + return .emptyMatchScore + } + } + } + + package func score( + candidate candidateMixedcaseBytes: UTF8Bytes, + contentType: ContentType, + precision: Precision, + allocator: inout UnsafeStackAllocator + ) -> TextScore { + if patternLowercaseBytes.hasContent { + var candidate = IndexedCandidate.allocate( + referencing: candidateMixedcaseBytes, + patternByteCount: patternLowercaseBytes.count, + firstMatchingLowercaseByteIndex: nil, + contentType: contentType, + allocator: &allocator + ) + defer { candidate.deallocate(allocator: &allocator) } + return score( + candidate: &candidate, + precision: precision, + captureMatchingRanges: false, + allocator: &allocator + ) + } else { + return .emptyMatchScore + } + } + + package func score( + candidate candidateMixedcaseBytes: UTF8Bytes, + contentType: ContentType, + precision: Precision, + captureMatchingRanges: Bool, + ranges: inout [UTF8ByteRange] + ) -> Double { + UnsafeStackAllocator.withUnsafeStackAllocator { allocator in + if patternLowercaseBytes.hasContent { + var candidate = IndexedCandidate.allocate( + referencing: candidateMixedcaseBytes, + patternByteCount: patternLowercaseBytes.count, + firstMatchingLowercaseByteIndex: nil, + contentType: contentType, + allocator: &allocator + ) + defer { candidate.deallocate(allocator: &allocator) } + let score = score( + candidate: &candidate, + precision: precision, + captureMatchingRanges: captureMatchingRanges, + allocator: &allocator + ) + if captureMatchingRanges { + ranges = Array(candidate.matchedRanges) + } + return score.value + } else { + return TextScore.emptyMatchScore.value + } + } + } + + private func score( + candidate: inout IndexedCandidate, + precision: Precision, + captureMatchingRanges: Bool, + allocator: inout UnsafeStackAllocator + ) -> TextScore { + switch precision { + case .fast: + let matchStyle = populateMatchingRangesForFastScoring(&candidate) + return singleScore(candidate: candidate, precision: precision, matchStyle: matchStyle) + case .thorough: + let budget = 5000 + return bestScore( + candidate: &candidate, + budget: budget, + captureMatchingRanges: captureMatchingRanges, + allocator: &allocator + ) + } + } + + private func eligibleForAcronymMatch(candidate: IndexedCandidate) -> Bool { + return (patternLowercaseBytes.count >= 3) + && candidate.contentType.isEligibleForAcronymMatch + && candidate.tokenization.hasNonUppercaseNonDelimiterBytes + } + + private enum MatchStyle: CaseIterable { + case lowercaseContinuous + case mixedcaseContinuous + case mixedcaseGreedy + case lowercaseGreedy + case acronym + } + + /// Looks for matches like `tamic` against `[t]ranslates[A]utoresizing[M]ask[I]nto[C]onstraints.` + /// Or `MOC` against `NS[M]anaged[O]bject[C]ontext`. + /// Accomplishes this by doing a greedy match over the acronym characters, which are the first characters of each token. + private func populateMatchingRangesForAcronymMatch(_ candidate: inout IndexedCandidate) -> Bool { + let tokenization = candidate.tokenization + if eligibleForAcronymMatch(candidate: candidate) && tokenization.tokens.hasContent { + let candidateLowercaseBytes = candidate.lowercaseBytes + let allowFullMatchesBeginningWithTokenIndex = + candidate.contentType.acronymMatchAllowsMultiCharacterMatchesAfterBaseName + ? candidate.tokenization.firstNonBaseNameTokenIndex : candidate.tokenization.tokenCount + var tidx = 0 + let tcnt = tokenization.tokens.count + var cidx = 0 + let ccnt = candidateLowercaseBytes.count + var pidx = 0 + let pcnt = patternLowercaseBytes.count + + var matches = true + while (cidx < ccnt) && (pidx < pcnt) && (tidx < tcnt) && matches { + let token = tokenization.tokens[tidx] + var tcidx = 0 + let tccnt = (token.allUppercase || (tidx >= allowFullMatchesBeginningWithTokenIndex)) ? token.length : 1 + let initialCidx = cidx + while (tcidx < tccnt) && (cidx < ccnt) && (pidx < pcnt) { + if candidateLowercaseBytes[cidx] == patternLowercaseBytes[pidx] { + tcidx += 1 + cidx += 1 + pidx += 1 + } else { + break + } + } + matches = + (tcidx > 0) // Matched any characters in this token + // Allow skipping the first token if it's all uppercase + || ((tidx == 0) && token.allUppercase) + // Allow skipping single character delemiters + || ((token.length == 1) && candidateLowercaseBytes[cidx].isDelimiter) + + if (tcidx > 0) { + candidate.matchedRanges.append(initialCidx.., + startOffset: Int, + candidateBytes: UTF8Bytes, + patternBytes: UTF8Bytes + ) -> Bool { + if let contiguousMatch = candidateBytes.rangeOf(bytes: patternBytes, startOffset: startOffset) { + ranges.append(contiguousMatch) + return true + } + return false + } + + /// Returns `true` if the pattern matches the candidate according to the rules of the `matchStyle`. + /// When successful, the matched ranges will be added to `ranges`. Otherwise `ranges` will be cleared. + private func populateMatchingRanges(_ candidate: inout IndexedCandidate, matchStyle: MatchStyle) -> Bool { + let startOffset = candidate.firstMatchingLowercaseByteIndex ?? 0 + switch matchStyle { + case .lowercaseContinuous: + return Self.populateMatchingContinuousRanges( + &candidate.matchedRanges, + startOffset: startOffset, + candidateBytes: candidate.lowercaseBytes, + patternBytes: patternLowercaseBytes + ) + case .mixedcaseContinuous: + return Self.populateMatchingContinuousRanges( + &candidate.matchedRanges, + startOffset: startOffset, + candidateBytes: candidate.mixedcaseBytes, + patternBytes: patternMixedcaseBytes + ) + case .acronym: + return populateMatchingRangesForAcronymMatch(&candidate) + case .mixedcaseGreedy: + return Self.populateGreedyMatchingRanges( + &candidate.matchedRanges, + startOffset: startOffset, + candidateBytes: candidate.mixedcaseBytes, + patternBytes: patternMixedcaseBytes + ) + case .lowercaseGreedy: + return Self.populateGreedyMatchingRanges( + &candidate.matchedRanges, + startOffset: startOffset, + candidateBytes: candidate.lowercaseBytes, + patternBytes: patternLowercaseBytes + ) + } + } + + /// Tries a fixed set of strategies in most to least desirable order for matching the candidate to the pattern. + /// + /// Returns the first strategy that matches. + /// Generally this match will produce the highest score of the considered strategies, but this isn't known, and a + /// higher scoring match could be found by the `.thorough` search. + /// For example, the highest priority strategy is a case matched contiguous sequence. If you search for "name" in + /// "filenames(name:)", this code will select the first one, but the second occurrence will score higher since it's a + /// complete token. + /// The fast scoring just needs to get the result into the second round though where `.thorough` will find the better + /// match. + private func populateMatchingRangesForFastScoring(_ candidate: inout IndexedCandidate) -> MatchStyle? { + let startOffset = candidate.firstMatchingLowercaseByteIndex ?? 0 + if Self.populateMatchingContinuousRanges( + &candidate.matchedRanges, + startOffset: startOffset, + candidateBytes: candidate.lowercaseBytes, + patternBytes: patternLowercaseBytes + ) { + return .lowercaseContinuous + } else if populateMatchingRangesForAcronymMatch(&candidate) { + return .acronym + } else if Self.populateGreedyMatchingRanges( + &candidate.matchedRanges, + startOffset: startOffset, + candidateBytes: candidate.mixedcaseBytes, + patternBytes: patternMixedcaseBytes + ) { + return .mixedcaseGreedy + } else if Self.populateGreedyMatchingRanges( + &candidate.matchedRanges, + startOffset: startOffset, + candidateBytes: candidate.lowercaseBytes, + patternBytes: patternLowercaseBytes + ) { + return .lowercaseGreedy + } + return nil + } + + /// Tries to match `patternBytes` against `candidateBytes` by greedily matching the first occurrence of a character + /// it finds, ignoring tokenization. For example, "filename" matches both "file(named:)" and "decoyfileadecoynamedecoy". + /// + /// If a successful match was found, returns `true` and populates `ranges` with the matched ranges. + /// Otherwise `ranges` is cleared since it's used as scratch space. + private static func populateGreedyMatchingRanges( + _ ranges: inout UnsafeStackArray, + startOffset: Int, + candidateBytes: UTF8Bytes, + patternBytes: UTF8Bytes + ) -> Bool { + var cidx = startOffset, pidx = 0 + var currentlyMatching = false + while (cidx < candidateBytes.count) && (pidx < patternBytes.count) { + if candidateBytes[cidx] == patternBytes[pidx] { + pidx += 1 + if !currentlyMatching { + currentlyMatching = true + ranges.append(cidx ..+ 0) + } + ranges[ranges.count - 1].extend(upperBoundBy: 1) + } else { + currentlyMatching = false + } + cidx += 1 + } + let matched = pidx == patternBytes.count + if !matched { + ranges.removeAll() + } + return matched + } + + private func singleScore(candidate: IndexedCandidate, precision: Precision, matchStyle: MatchStyle?) -> TextScore { + func ratio(_ lhs: Int, _ rhs: Int) -> Double { + return Double(lhs) / Double(rhs) + } + + var patternCharactersRemaining = patternMixedcaseBytes.count + let matchedRanges = candidate.matchedRanges + if let firstMatchedRange = matchedRanges.first { + let candidateMixedcaseBytes = candidate.mixedcaseBytes + let tokenization = candidate.tokenization + let prefixMatchBonusValue = candidate.contentType.prefixMatchBonus + let contentAfterBasenameIsTrivial = candidate.contentType.contentAfterBasenameIsTrivial + let leadingCaseMatchableCount = + contentAfterBasenameIsTrivial ? tokenization.baseNameLength : candidateMixedcaseBytes.count + var score = 0.0 + var falseStarts = 0 + var uppercaseMatches = 0 + var uppercaseMismatches = 0 + var anyCaseMatches = 0 + var isPrefixUppercaseMatch = false + do { + var pidx = 0 + for matchedRange in matchedRanges { + for cidx in matchedRange { + let candidateCharacter = candidateMixedcaseBytes[cidx] + if cidx < leadingCaseMatchableCount { + if candidateCharacter == patternMixedcaseBytes[pidx] { + /// Check for case match. + uppercaseMatches += candidateCharacter.isUppercase ? 1 : 0 + isPrefixUppercaseMatch = + isPrefixUppercaseMatch || (candidateCharacter.isUppercase && (cidx == 0)) + anyCaseMatches += 1 + } else { + uppercaseMismatches += 1 + } + } + pidx += 1 + } + } + } + + var badShortMatches = 0 + var incompletelyMatchedTokens = 0 + var allRunsStartOnWordStartOrUppercaseLetter = true + + for range in matchedRanges { + var position = range.lowerBound + var remainingCharacters = range.length + var matchedTokenPrefix = false + repeat { + let tokenIndex = tokenization.byteTokenAddresses[position].tokenIndex + let tokenLength = tokenization.tokens[tokenIndex].length + let positionInToken = tokenization.byteTokenAddresses[position].indexInToken + let tokenCharactersRemaining = (tokenLength - positionInToken) + let coveredCharacters = + (remainingCharacters > tokenCharactersRemaining) + ? tokenCharactersRemaining : remainingCharacters + let coveredWholeToken = (coveredCharacters == tokenLength) + incompletelyMatchedTokens += coveredWholeToken ? 0 : 1 + let laterMatchesExist = (coveredCharacters < patternCharactersRemaining) + + let incompleteMatch = (!coveredWholeToken && laterMatchesExist) + if incompleteMatch || (positionInToken != 0) { + falseStarts += 1 + } + + if incompleteMatch && (coveredCharacters <= 2) { + badShortMatches += 1 + } + if positionInToken == 0 { + matchedTokenPrefix = true + } else if !candidateMixedcaseBytes[position].isUppercase { + allRunsStartOnWordStartOrUppercaseLetter = false + } + + patternCharactersRemaining -= coveredCharacters + remainingCharacters -= coveredCharacters + position += coveredCharacters + } while (remainingCharacters > 0) + if (range.length > 1) || matchedTokenPrefix { + score += pow(Double(range.length), 1.5) + } + } + // This is for cases like an autogenerated member-wise initializer of a huge struct matching everything. + // If they only matched within the arguments, and it's a huge symbol, it's a false start. + if (firstMatchedRange.lowerBound > tokenization.baseNameLength) && (candidateMixedcaseBytes.count > 256) { + falseStarts += 1 + score *= 0.75 + } + + if (matchStyle == .acronym) { + badShortMatches = 0 + falseStarts = 0 + } + + if matchedRanges.only?.length == candidateMixedcaseBytes.count { + score *= candidate.contentType.fullMatchBonus + } else if matchedRanges.only == 0.. 1) && (matchStyle != .acronym) { + score /= 2 + } + } + return TextScore(value: score, falseStarts: falseStarts) + } else { + return .noMatchScore + } + } + + private static func allocateSuccessiveRejectionFilters( + lowercaseBytes: UTF8Bytes, + allocator: inout UnsafeStackAllocator + ) -> UnsafeStackArray { + var filters = allocator.allocateUnsafeArray(of: RejectionFilter.self, maximumCapacity: lowercaseBytes.count) + filters.initializeWithContainedGarbage() + var idx = lowercaseBytes.count - 1 + var accumulated = RejectionFilter.empty + while idx >= 0 { + accumulated.formUnion(lowercaseByte: lowercaseBytes[idx]) + filters[idx] = accumulated + idx -= 1 + } + return filters + } +} + +extension Pattern { + package struct ByteTokenAddress { + package var tokenIndex = 0 + package var indexInToken = 0 + } + + /// A token is a single conceptual name piece of an identifier, usually separated by camel case or underscores. + /// + /// For example `doFancyStuff` is divided into 3 tokens: `do`, `Fancy`, and `Stuff`. + package struct Token { + var storage: UInt + init(length: Int, allUppercase: Bool) { + let bit: UInt = (allUppercase ? 1 : 0) << (Int.bitWidth - 1) + storage = UInt(bitPattern: length) | bit + } + + package var length: Int { + Int(bitPattern: UInt(storage & ~(1 << (Int.bitWidth - 1)))) + } + + package var allUppercase: Bool { + (storage & (1 << (Int.bitWidth - 1))) != 0 + } + } + + package struct Tokenization { + package private(set) var baseNameLength: Int + package private(set) var hasNonUppercaseNonDelimiterBytes: Bool + package private(set) var tokens: UnsafeStackArray + package private(set) var byteTokenAddresses: UnsafeStackArray + + var tokenCount: Int { + tokens.count + } + + var byteCount: Int { + byteTokenAddresses.count + } + + private enum CharacterClass: Equatable { + case uppercase + case delimiter + case other + } + + // `nonisolated(unsafe)` is fine because the underlying buffer is never mutated or deallocated. + private nonisolated(unsafe) static let characterClasses: UnsafePointer = { + let array: [CharacterClass] = (0...255).map { (character: UInt8) in + if character.isDelimiter { + return .delimiter + } else if character.isUppercase { + return .uppercase + } else { + return .other + } + } + return UnsafeBufferPointer.allocate(copyOf: array).baseAddress! + }() + + // name -> [name] + // myName -> [my][Name] + // File.h -> [File][.][h] + // NSOpenGLView -> [NS][Open][GL][View] + // NSURL -> [NSURL] + private init(mixedcaseBytes: UTF8Bytes, contentType: ContentType, allocator: inout UnsafeStackAllocator) { + let byteCount = mixedcaseBytes.count + let maxTokenCount = byteCount + let baseNameMatchesLast = (contentType.baseNameAffinity == .last) + let baseNameSeparator = contentType.baseNameSeparator + byteTokenAddresses = allocator.allocateUnsafeArray(of: ByteTokenAddress.self, maximumCapacity: byteCount) + tokens = allocator.allocateUnsafeArray(of: Token.self, maximumCapacity: maxTokenCount) + baseNameLength = -1 + var baseNameLength: Int? = nil + if byteCount > 1 { + let mixedcaseBytes = mixedcaseBytes.baseAddress! + let characterClasses = Self.characterClasses + let endMixedCaseBytes = mixedcaseBytes + byteCount + func characterClass(at pointer: UnsafePointer) -> CharacterClass { + if pointer != endMixedCaseBytes { + return characterClasses[Int(pointer.pointee)] + } else { + return .delimiter + } + } + var previous = characterClass(at: mixedcaseBytes) + var current = characterClass(at: mixedcaseBytes + 1) + var token = (index: 0, length: 1, isAllUppercase: (previous == .uppercase)) + hasNonUppercaseNonDelimiterBytes = (previous == .other) + tokens.initializeWithContainedGarbage() + byteTokenAddresses.initializeWithContainedGarbage() + var nextByteTokenAddress = byteTokenAddresses.base + nextByteTokenAddress.pointee = .init(tokenIndex: 0, indexInToken: 0) + var nextBytePointer = mixedcaseBytes + 1 + while nextBytePointer != endMixedCaseBytes { + nextBytePointer += 1 + let next = characterClass(at: nextBytePointer) + let currentIsUppercase = (current == .uppercase) + let tokenizeBeforeCurrentCharacter = + (currentIsUppercase && ((previous == .other) || (next == .other))) + || (current == .delimiter) + || (previous == .delimiter) + + if tokenizeBeforeCurrentCharacter { + let anyOtherCase = !(token.isAllUppercase || (previous == .delimiter)) + hasNonUppercaseNonDelimiterBytes = hasNonUppercaseNonDelimiterBytes || anyOtherCase + tokens[token.index] = .init(length: token.length, allUppercase: token.isAllUppercase) + token.isAllUppercase = true + token.length = 0 + token.index += 1 + let lookBack = nextBytePointer - 2 + if lookBack.pointee == baseNameSeparator { + if baseNameLength == nil || baseNameMatchesLast { + baseNameLength = mixedcaseBytes.distance(to: lookBack) + } + } + } + token.isAllUppercase = token.isAllUppercase && currentIsUppercase + nextByteTokenAddress += 1 + nextByteTokenAddress.pointee.tokenIndex = token.index + nextByteTokenAddress.pointee.indexInToken = token.length + token.length += 1 + previous = current + current = next + } + let anyOtherCase = !(token.isAllUppercase || (previous == .delimiter)) + hasNonUppercaseNonDelimiterBytes = hasNonUppercaseNonDelimiterBytes || anyOtherCase + tokens[token.index] = .init(length: token.length, allUppercase: token.isAllUppercase) + tokens.truncateLeavingGarbage(to: token.index + 1) + } else if byteCount == 1 { + let characterClass = Self.characterClasses[Int(mixedcaseBytes[0])] + tokens.append(.init(length: 1, allUppercase: characterClass == .uppercase)) + byteTokenAddresses.append(.init(tokenIndex: 0, indexInToken: 0)) + hasNonUppercaseNonDelimiterBytes = (characterClass == .other) + } else { + hasNonUppercaseNonDelimiterBytes = false + } + self.baseNameLength = baseNameLength ?? byteCount + } + + func enumerate(body: (Range) -> ()) { + var position = 0 + for token in tokens { + body(position ..+ token.length) + position += token.length + } + } + + func anySatisfy(predicate: (Range) -> Bool) -> Bool { + var position = 0 + for token in tokens { + if predicate(position ..+ token.length) { + return true + } + position += token.length + } + return false + } + + package mutating func deallocate(allocator: inout UnsafeStackAllocator) { + allocator.deallocate(&tokens) + allocator.deallocate(&byteTokenAddresses) + } + + package static func allocate( + mixedcaseBytes: UTF8Bytes, + contentType: ContentType, + allocator: inout UnsafeStackAllocator + ) + -> Tokenization + { + Tokenization(mixedcaseBytes: mixedcaseBytes, contentType: contentType, allocator: &allocator) + } + + var firstNonBaseNameTokenIndex: Int { + (byteCount == baseNameLength) ? tokenCount : byteTokenAddresses[baseNameLength].tokenIndex + } + } +} + +extension Pattern.UTF8Bytes { + fileprivate func allocateLowercaseBytes(allocator: inout UnsafeStackAllocator) -> Self { + let lowercaseBytes = allocator.allocateBuffer(of: UTF8Byte.self, count: count) + for index in indices { + lowercaseBytes[index] = self[index].lowercasedUTF8Byte + } + return UnsafeBufferPointer(lowercaseBytes) + } +} + +extension Pattern { + fileprivate struct IndexedCandidate { + var mixedcaseBytes: UTF8Bytes + var lowercaseBytes: UTF8Bytes + var contentType: ContentType + var tokenization: Tokenization + var matchedRanges: UnsafeStackArray + var firstMatchingLowercaseByteIndex: Int? + + static func allocate( + referencing mixedcaseBytes: UTF8Bytes, + patternByteCount: Int, + firstMatchingLowercaseByteIndex: Int?, + contentType: ContentType, + allocator: inout UnsafeStackAllocator + ) -> Self { + let lowercaseBytes = mixedcaseBytes.allocateLowercaseBytes(allocator: &allocator) + let tokenization = Tokenization.allocate( + mixedcaseBytes: mixedcaseBytes, + contentType: contentType, + allocator: &allocator + ) + let matchedRanges = allocator.allocateUnsafeArray(of: UTF8ByteRange.self, maximumCapacity: patternByteCount) + return Self( + mixedcaseBytes: mixedcaseBytes, + lowercaseBytes: lowercaseBytes, + contentType: contentType, + tokenization: tokenization, + matchedRanges: matchedRanges, + firstMatchingLowercaseByteIndex: firstMatchingLowercaseByteIndex + ) + } + + mutating func deallocate(allocator: inout UnsafeStackAllocator) { + allocator.deallocate(&matchedRanges) + tokenization.deallocate(allocator: &allocator) + allocator.deallocate(&lowercaseBytes) + } + + /// Create a new stack array such that for every offset `i` in this pattern `result[i]` points to the start offset + /// of the next token. If there are no more valid start locations `result[i]` points to one character after the end + /// of the candidate. + /// Valid start locations are the start of the next token that begins with a byte that's in the pattern. + /// + /// Examples: + /// ------------------------------------------------------------------------------------------------------- + /// Candidate | Pattern | Result + /// ------------------------------------------------------------------------------------------------------- + /// "doSomeWork" | "SWork" | `[2, 2, 6, 6, 6, 6, 10, 10, 10, 10]` + /// "fn(one:two:three:four:)" | "tf" | `[7,7,7,7,7,7,7,11,11,11,11,17,17,17,17,17,17,23,23,23,23,23,23]` + /// + fileprivate func allocateNextSearchStarts( + allocator: inout UnsafeStackAllocator, + patternRejectionFilter: RejectionFilter + ) -> UnsafeStackArray { + var nextSearchStarts = allocator.allocateUnsafeArray(of: Int.self, maximumCapacity: mixedcaseBytes.count) + var nextStart = mixedcaseBytes.count + let byteTokenAddresses = tokenization.byteTokenAddresses + nextSearchStarts.initializeWithContainedGarbage() + for cidx in mixedcaseBytes.indices.reversed() { + nextSearchStarts[cidx] = nextStart + let isTokenStart = byteTokenAddresses[cidx].indexInToken == 0 + if isTokenStart && patternRejectionFilter.contains(candidateByte: mixedcaseBytes[cidx]) == .maybe { + nextStart = cidx + } + } + return nextSearchStarts + } + + var looksLikeAType: Bool { + (tokenization.hasNonUppercaseNonDelimiterBytes && tokenization.baseNameLength == mixedcaseBytes.count) + } + } +} + +// MARK: - Best Score Search - + +extension Pattern { + private struct Location { + var pattern: Int + var candidate: Int + + func nextByMatching() -> Self { + Location(pattern: pattern + 1, candidate: candidate + 1) + } + + func nextBySkipping(validStartingLocations: UnsafeStackArray) -> Self { + Location(pattern: pattern, candidate: validStartingLocations[candidate]) + } + + static let start = Self(pattern: 0, candidate: 0) + } + + private enum RestoredRange { + case restore(UTF8ByteRange) + case unwind + case none + + func restore(ranges: inout UnsafeStackArray) { + switch self { + case .restore(let lastRange): + ranges[ranges.count - 1] = lastRange + case .unwind: + ranges.removeLast() + case .none: + break + } + } + } + + private struct Step { + var location: Location + var restoredRange: RestoredRange + } + + private struct Context { + var pattern: UTF8Bytes + var candidate: UTF8Bytes + var location = Location.start + + var patternBytesRemaining: Int { + pattern.count - location.pattern + } + + var candidateBytesRemaining: Int { + candidate.count - location.candidate + } + + var enoughCandidateBytesRemain: Bool { + patternBytesRemaining <= candidateBytesRemaining + } + + var isCompleteMatch: Bool { + return patternBytesRemaining == 0 + } + + var isCharacterMatch: Bool { + pattern[location.pattern] == candidate[location.candidate] + } + } + + private struct Buffers { + var steps: UnsafeStackArray + var bestRangeSnapshot: UnsafeStackArray + var validMatchStartingLocations: UnsafeStackArray + /// For each byte the candidate's text, a rejection filter that contains all the characters occurring after or + /// at that offset. + /// + /// This way when we have already matched the first 4 bytes, we can check which characters occur from byte 5 + /// onwards and check that they appear in the candidate's remaining text. + var candidateSuccessiveRejectionFilters: UnsafeStackArray + + static func allocate( + patternLowercaseBytes: UTF8Bytes, + candidate: IndexedCandidate, + patternRejectionFilter: RejectionFilter, + allocator: inout UnsafeStackAllocator + ) -> Self { + let steps = allocator.allocateUnsafeArray(of: Step.self, maximumCapacity: patternLowercaseBytes.count + 1) + let bestRangeSnapshot = allocator.allocateUnsafeArray( + of: UTF8ByteRange.self, + maximumCapacity: patternLowercaseBytes.count + ) + let validMatchStartingLocations = candidate.allocateNextSearchStarts( + allocator: &allocator, + patternRejectionFilter: patternRejectionFilter + ) + let candidateSuccessiveRejectionFilters = Pattern.allocateSuccessiveRejectionFilters( + lowercaseBytes: candidate.lowercaseBytes, + allocator: &allocator + ) + return Buffers( + steps: steps, + bestRangeSnapshot: bestRangeSnapshot, + validMatchStartingLocations: validMatchStartingLocations, + candidateSuccessiveRejectionFilters: candidateSuccessiveRejectionFilters + ) + } + + mutating func deallocate(allocator: inout UnsafeStackAllocator) { + allocator.deallocate(&candidateSuccessiveRejectionFilters) + allocator.deallocate(&validMatchStartingLocations) + allocator.deallocate(&bestRangeSnapshot) + allocator.deallocate(&steps) + } + } + + private struct ExecutionLimit { + private(set) var remainingCycles: Int + mutating func permitCycle() -> Bool { + if remainingCycles == 0 { + return false + } else { + remainingCycles -= 1 + return true + } + } + } + + /// Exhaustively searches for ways to match the pattern to the candidate, scoring each one, and then returning the best score. + /// If `captureMatchingRanges` is `true`, and a match is found, writes the matching ranges out to `matchedRangesStorage`. + fileprivate func bestScore( + candidate: inout IndexedCandidate, + budget maxSteps: Int, + captureMatchingRanges: Bool, + allocator: inout UnsafeStackAllocator + ) -> TextScore { + var context = Context(pattern: patternLowercaseBytes, candidate: candidate.lowercaseBytes) + var bestScore: TextScore? = nil + // Find the best score by visiting all possible scorings. + // So given the pattern "down" and candidate "documentDocument", enumerate these matches: + // - [do]cumentDo[wn]load + // - [d]ocumentD[own]load + // - document[Down]load + // Score each one, and choose the best. + // + // While recursion would be easier, use a manual stack because in practice we run out of stack space. + // + // Shortcuts: + // * Stop if there are more character remaining to be matched than candidates to match. + // * Keep a list of successive reject filters for the candidate and pattern, so that if the remaining pattern + // characters are known not to exist in the rest of the candidate, we can stop early. + // * Each time we step forward after a failed match in a candidate, move up to the next token, not next character. + // * Impose a budget of several thousand iterations so that we back off if things are going to hit the exponential worst case + if patternMixedcaseBytes.hasContent && context.enoughCandidateBytesRemain { + var buffers = Buffers.allocate( + patternLowercaseBytes: patternLowercaseBytes, + candidate: candidate, + patternRejectionFilter: patternRejectionFilter, + allocator: &allocator + ) + defer { buffers.deallocate(allocator: &allocator) } + var executionLimit = ExecutionLimit(remainingCycles: maxSteps) + buffers.steps.push(Step(location: .start, restoredRange: .none)) + while executionLimit.permitCycle(), let step = buffers.steps.popLast() { + context.location = step.location + step.restoredRange.restore(ranges: &candidate.matchedRanges) + + if context.isCompleteMatch { + let newScore = singleScore(candidate: candidate, precision: .thorough, matchStyle: nil) + accumulate( + score: newScore, + into: &bestScore, + matchedRanges: candidate.matchedRanges, + captureMatchingRanges: captureMatchingRanges, + matchedRangesStorage: &buffers.bestRangeSnapshot + ) + } else if context.enoughCandidateBytesRemain { + if RejectionFilter.match( + pattern: patternSuccessiveRejectionFilters[context.location.pattern], + candidate: buffers.candidateSuccessiveRejectionFilters[context.location.candidate] + ) == .maybe { + if context.isCharacterMatch { + let extending = candidate.matchedRanges.last?.upperBound == context.location.candidate + let restoredRange: RestoredRange + if extending { + let lastIndex = candidate.matchedRanges.count - 1 + restoredRange = .restore(candidate.matchedRanges[lastIndex]) + candidate.matchedRanges[lastIndex].extend(upperBoundBy: 1) + } else { + restoredRange = .unwind + candidate.matchedRanges.append(context.location.candidate ..+ 1) + } + buffers.steps.push( + Step( + location: context.location.nextBySkipping( + validStartingLocations: buffers.validMatchStartingLocations + ), + restoredRange: restoredRange + ) + ) + buffers.steps.push(Step(location: context.location.nextByMatching(), restoredRange: .none)) + } else { + buffers.steps.push( + Step( + location: context.location.nextBySkipping( + validStartingLocations: buffers.validMatchStartingLocations + ), + restoredRange: .none + ) + ) + } + } + } + } + + // We need to consider the special cases the `.fast` search scans for so that a `.fast` score can't be better than + // a .`thorough` score. + // For example, they could match 30 character that weren't on a token boundary, which we would have skipped above. + // Or we could have ran out of time searching and missed a contiguous match. + for matchStyle in MatchStyle.allCases { + candidate.matchedRanges.removeAll() + if populateMatchingRanges(&candidate, matchStyle: matchStyle) { + let newScore = singleScore(candidate: candidate, precision: .thorough, matchStyle: matchStyle) + accumulate( + score: newScore, + into: &bestScore, + matchedRanges: candidate.matchedRanges, + captureMatchingRanges: captureMatchingRanges, + matchedRangesStorage: &buffers.bestRangeSnapshot + ) + } + } + candidate.matchedRanges.removeAll() + candidate.matchedRanges.append(contentsOf: buffers.bestRangeSnapshot) + } + return bestScore ?? .noMatchScore + } + + private func accumulate( + score newScore: TextScore, + into bestScore: inout TextScore?, + matchedRanges: UnsafeStackArray, + captureMatchingRanges: Bool, + matchedRangesStorage bestMatchedRangesStorage: inout UnsafeStackArray + ) { + if newScore > (bestScore ?? .worstPossibleScore) { + bestScore = newScore + if captureMatchingRanges { + bestMatchedRangesStorage.removeAll() + bestMatchedRangesStorage.append(contentsOf: matchedRanges) + } + } + } +} + +extension Pattern { + package func test_searchStart(candidate: Candidate, contentType: ContentType) -> [Int] { + UnsafeStackAllocator.withUnsafeStackAllocator { allocator in + var indexedCandidate = IndexedCandidate.allocate( + referencing: candidate.bytes, + patternByteCount: patternLowercaseBytes.count, + firstMatchingLowercaseByteIndex: nil, + contentType: contentType, + allocator: &allocator + ) + defer { indexedCandidate.deallocate(allocator: &allocator) } + var nextSearchStarts = indexedCandidate.allocateNextSearchStarts( + allocator: &allocator, + patternRejectionFilter: patternRejectionFilter + ) + defer { allocator.deallocate(&nextSearchStarts) } + return Array(nextSearchStarts) + } + } + + package func testPerformance_tokenizing(batch: CandidateBatch, contentType: ContentType) -> Int { + UnsafeStackAllocator.withUnsafeStackAllocator { allocator in + var all = 0 + batch.enumerate { candidate in + var indexedCandidate = IndexedCandidate.allocate( + referencing: candidate.bytes, + patternByteCount: patternLowercaseBytes.count, + firstMatchingLowercaseByteIndex: nil, + contentType: contentType, + allocator: &allocator + ) + defer { indexedCandidate.deallocate(allocator: &allocator) } + all &= indexedCandidate.tokenization.tokenCount + } + return all + } + } +} + +extension Candidate.ContentType { + fileprivate var prefixMatchBonus: Double { + switch self { + case .codeCompletionSymbol: return 2.00 + case .fileName, .projectSymbol: return 1.05 + case .unknown: return 2.00 + } + } + + fileprivate var fullMatchBonus: Double { + switch self { + case .codeCompletionSymbol: return 1.00 + case .fileName, .projectSymbol: return 1.50 + case .unknown: return 1.00 + } + } + + fileprivate var fullBaseNameMatchBonus: Double { + switch self { + case .codeCompletionSymbol: return 1.00 + case .fileName, .projectSymbol: return 1.50 + case .unknown: return 1.00 + } + } + + fileprivate var baseNameAffinity: BaseNameAffinity { + switch self { + case .codeCompletionSymbol, .projectSymbol: return .first + case .fileName: return .last + case .unknown: return .last + } + } + + fileprivate var baseNameSeparator: UTF8Byte { + switch self { + case .codeCompletionSymbol, .projectSymbol: return .cLeftParentheses + case .fileName: return .cPeriod + case .unknown: return 0 + } + } + + fileprivate var isEligibleForAcronymMatch: Bool { + switch self { + case .codeCompletionSymbol, .fileName, .projectSymbol: return true + case .unknown: return false + } + } + + fileprivate var acronymMatchAllowsMultiCharacterMatchesAfterBaseName: Bool { + switch self { + // You can't do a acronym match into function arguments + case .codeCompletionSymbol, .projectSymbol, .unknown: return false + case .fileName: return true + } + } + + fileprivate var acronymMatchMustBeInBaseName: Bool { + switch self { + // You can't do a acronym match into function arguments + case .codeCompletionSymbol, .projectSymbol: return true + case .fileName, .unknown: return false + } + } + + fileprivate var contentAfterBasenameIsTrivial: Bool { + switch self { + case .codeCompletionSymbol, .projectSymbol: return false + case .fileName: return true + case .unknown: return false + } + } + + fileprivate var isEligibleForTypeNameOverLocalVariableModifier: Bool { + switch self { + case .codeCompletionSymbol: return true + case .fileName, .projectSymbol, .unknown: return false + } + } + + fileprivate enum BaseNameAffinity { + case first + case last + } +} + +extension Pattern { + @available(*, deprecated, message: "Pass a contentType") + package func score( + candidate: UTF8Bytes, + precision: Precision, + captureMatchingRanges: Bool, + ranges: inout [UTF8ByteRange] + ) -> Double { + score( + candidate: candidate, + contentType: .codeCompletionSymbol, + precision: precision, + captureMatchingRanges: captureMatchingRanges, + ranges: &ranges + ) + } + + @available(*, deprecated, message: "Pass a contentType") + package func score(candidate: UTF8Bytes, precision: Precision) -> Double { + score(candidate: candidate, contentType: .codeCompletionSymbol, precision: precision) + } +} diff --git a/Sources/CompletionScoring/Text/RejectionFilter.swift b/Sources/CompletionScoring/Text/RejectionFilter.swift new file mode 100644 index 000000000..ee08aceff --- /dev/null +++ b/Sources/CompletionScoring/Text/RejectionFilter.swift @@ -0,0 +1,135 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +/// A `RejectionFilter` can quickly rule out two byte strings matching with a simple bitwise and. +/// It's the first, most aggressive, and cheapest filter used in matching. +/// +/// -- The Mask -- +/// It's 32 bits. Conceptually it uses 26 bits to represent `a...z`. If the letter corresponding +/// to a byte was present in the candidate, it's bit is set in the filter. +/// +/// The filter is case insensitive, so making a filter from "abc" produces the same filter as one +/// produced from "AbC". +/// +/// Computing ranges in the alphabet is a bottle neck, and a filter will work as long as it's case +/// insensitive for ascii. So instead of mapping `a...z` to `0...25` we map the lower 5 bits of the +/// byte to (0...31), and then shift 1 by that amount to map all bytes to a bit in a way that is +/// both case preserving, and very cheap to compute. +/// +/// -- The Filter -- +/// For a pattern to match a candidate, all bytes in the pattern must be in the candidate in the +/// same order - ascii case insensitive. If you have a mask for the pattern and candidate, then +/// if any bits from the pattern mask aren't in the candidate mask, they can't possibly satisfy +/// the stricter matching criteria. +/// +/// If every bit in the pattern mask is also in the candidate mask it might match the candidate. +/// Examples of cases where it still wouldn't match: +/// * Character occurs 2 times in pattern, but 1 time in candidate +/// * Character in pattern are in different order from candidate +/// * Multiple distinct pattern characters mapped to the same bit, and only one of them was +/// in the candidate. For example, both '8' and 'x' map to the same bit. +package struct RejectionFilter { + package enum Match { + case no + case maybe + } + + private var mask: UInt32 = 0 + static var empty: Self { + .init(mask: 0) + } + + private func maskBit(byte: UTF8Byte) -> UInt32 { + // This mapping relies on the fact that the ascii values for a...z and A...Z + // are equivalent modulo 32, and since that's a power of 2, we can extract + // a bunch of information about that with shifts and masks. + // The comments below refer to each of these groups of 32 values as "bands" + + // The last 5 bits of the byte fulfill the following properties: + // - Every character in a...z has a unique value + // - The value of an uppercase character is the same as the corresponding lowercase character + let lowerFiveBits = UInt32(1) << UInt32(byte & 0b0001_1111) + + // We want to minimize aliasing between a-z values with other values with the same lower 5 bits. + // Start with their 6th bit, which will be zero for many of the non-alpha bands. + let hasAlphaBandBit = UInt32(byte & 0b0100_0000) >> 6 + + // Multiply their lower five bits by that value to map them to either themselves, or 0. + // This eliminates aliasing between 'z' and ':', 'h' and '(', and ')' and 'i', which commonly + // occur in filter text. + let mask = lowerFiveBits * hasAlphaBandBit + + // Ensure that every byte sets at least one bit, by always setting 0b01. + // That bit is never set for a-z because a is the second character in its band. + // + // Technically we don't need this, but without it you get surprising but not wrong results like + // all characters outside of the alpha bands return `.maybe` for matching either the empty string, + // or characters inside the alpha bands. + return mask | 0b01 + } + + package init(bytes: Bytes) where Bytes.Element == UTF8Byte { + for byte in bytes { + mask = mask | maskBit(byte: byte) + } + } + + package init(lowercaseBytes: Bytes) where Bytes.Element == UTF8Byte { + for byte in lowercaseBytes { + mask = mask | maskBit(byte: byte) + } + } + + private init(mask: UInt32) { + self.mask = mask + } + + package init(lowercaseByte: UTF8Byte) { + mask = maskBit(byte: lowercaseByte) + } + + package init(string: String) { + self.init(bytes: string.utf8) + } + + package static func match(pattern: RejectionFilter, candidate: RejectionFilter) -> Match { + return (pattern.mask & candidate.mask) == pattern.mask ? .maybe : .no + } + + func contains(candidateByte: UTF8Byte) -> Match { + let candidateMask = maskBit(byte: candidateByte) + return (mask & candidateMask) == candidateMask ? .maybe : .no + } + + mutating func formUnion(_ rhs: Self) { + mask = mask | rhs.mask + } + + mutating func formUnion(lowercaseByte: UTF8Byte) { + mask = mask | maskBit(byte: lowercaseByte) + } +} + +extension RejectionFilter: CustomStringConvertible { + package var description: String { + let base = String(mask, radix: 2) + return "0b" + String(repeating: "0", count: mask.bitWidth - base.count) + base + } +} + +extension RejectionFilter: Equatable { + package static func == (lhs: RejectionFilter, rhs: RejectionFilter) -> Bool { + lhs.mask == rhs.mask + } +} diff --git a/Sources/CompletionScoring/Text/ScoredMatchSelector.swift b/Sources/CompletionScoring/Text/ScoredMatchSelector.swift new file mode 100644 index 000000000..01a422859 --- /dev/null +++ b/Sources/CompletionScoring/Text/ScoredMatchSelector.swift @@ -0,0 +1,119 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +/// Use `ScoredMatchSelector` to find the matching indexes for a `Pattern` from an array of `CandidateBatch` structures. +/// It's a reference type to allow sharing state between calls to improve performance, and has an internal lock, +/// so only enter it from one thread at a time. +/// +/// It's primary performance improvement is that it creates and initializes all of the scratch buffers ahead of time, so that they can be +/// amortized across all matching calls. +package class ScoredMatchSelector { + typealias CandidateBatchSlice = Pattern.ScoringWorkload.CandidateBatchSlice + typealias CandidateBatchesMatch = Pattern.CandidateBatchesMatch + typealias ScoringWorkload = Pattern.ScoringWorkload + + private let threadWorkloads: [ThreadWorkload] + private let queue: DispatchQueue + package init(batches: [CandidateBatch]) { + let scoringWorkloads = ScoringWorkload.workloads( + for: batches, + parallelism: ProcessInfo.processInfo.processorCount + ) + threadWorkloads = scoringWorkloads.map { scoringWorkload in + ThreadWorkload(allBatches: batches, slices: scoringWorkload.slices) + } + queue = DispatchQueue(label: "ScoredMatchSelector") + } + + /// Find all of the matches across `batches` and score them, returning the scored results. This is a first part of selecting matches. Later the matches will be combined with matches from other providers, where we'll pick the best matches and sort them with `selectBestMatches(from:textProvider:)` + package func scoredMatches(pattern: Pattern, precision: Pattern.Precision) -> [Pattern.CandidateBatchesMatch] { + /// The whole point is share the allocation space across many re-uses, but if the client calls into us concurrently, we shouldn't just crash. An assert would also be acceptable. + return queue.sync { + // `nonisolated(unsafe)` is fine because every concurrent iteration accesses a different element. + nonisolated(unsafe) let threadWorkloads = threadWorkloads + DispatchQueue.concurrentPerform(iterations: threadWorkloads.count) { index in + threadWorkloads[index].updateOutput(pattern: pattern, precision: precision) + } + let totalMatchCount = threadWorkloads.sum { threadWorkload in + threadWorkload.matchCount + } + return Array(unsafeUninitializedCapacity: totalMatchCount) { aggregate, initializedCount in + if var writePosition = aggregate.baseAddress { + for threadWorkload in threadWorkloads { + threadWorkload.moveResults(to: &writePosition) + } + } else { + precondition(totalMatchCount == 0) + } + initializedCount = totalMatchCount + } + } + } +} + +extension ScoredMatchSelector { + fileprivate final class ThreadWorkload { + + fileprivate private(set) var matchCount = 0 + private let output: UnsafeMutablePointer + private let slices: [CandidateBatchSlice] + private let allBatches: [CandidateBatch] + + init(allBatches: [CandidateBatch], slices: [CandidateBatchSlice]) { + let candidateCount = slices.sum { slice in + slice.candidateRange.count + } + self.output = UnsafeMutablePointer.allocate(capacity: candidateCount) + self.slices = slices + self.allBatches = allBatches + } + + deinit { + precondition(matchCount == 0) // Missing call to moveResults? + output.deallocate() + } + + fileprivate func updateOutput(pattern: Pattern, precision: Pattern.Precision) { + precondition(matchCount == 0) // Missing call to moveResults? + self.matchCount = UnsafeStackAllocator.withUnsafeStackAllocator { allocator in + var matchCount = 0 + for slice in slices { + let batch = allBatches[slice.batchIndex] + batch.enumerate(slice.candidateRange) { candidateIndex, candidate in + if let score = pattern.matchAndScore( + candidate: candidate, + precision: precision, + allocator: &allocator + ) { + let match = CandidateBatchesMatch( + batchIndex: slice.batchIndex, + candidateIndex: candidateIndex, + textScore: score.value + ) + output.advanced(by: matchCount).initialize(to: match) + matchCount += 1 + } + } + } + return matchCount + } + } + + fileprivate func moveResults(to aggregate: inout UnsafeMutablePointer) { + aggregate.moveInitialize(from: output, count: matchCount) + aggregate = aggregate.advanced(by: matchCount) + matchCount = 0 + } + } +} diff --git a/Sources/CompletionScoring/Text/UTF8Byte.swift b/Sources/CompletionScoring/Text/UTF8Byte.swift new file mode 100644 index 000000000..d91b5aded --- /dev/null +++ b/Sources/CompletionScoring/Text/UTF8Byte.swift @@ -0,0 +1,65 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +package typealias UTF8Byte = UInt8 +package extension UTF8Byte { + init(_ character: Character) throws { + self = try character.utf8.only.unwrap(orThrow: "More than one byte: \(character)") + } +} + +package func UTF8ByteValue(_ character: Character) -> UTF8Byte? { + character.utf8.only +} + +package extension UTF8Byte { + static let uppercaseAZ: ClosedRange = (65...90) + static let lowercaseAZ: ClosedRange = (97...122) + + static let cSpace: Self = 32 // ' ' + static let cPlus: Self = 43 // '+' + static let cMinus: Self = 45 // '-' + static let cColon: Self = 58 // ':' + static let cPeriod: Self = 46 // '.' + static let cLeftParentheses: Self = 40 // '(' + static let cRightParentheses: Self = 41 // ')' + static let cUnderscore: Self = 95 // '_' + + var isLowercase: Bool { + return Self.lowercaseAZ.contains(self) + } + + var isUppercase: Bool { + return Self.uppercaseAZ.contains(self) + } + + var lowercasedUTF8Byte: UInt8 { + return isUppercase ? (self - Self.uppercaseAZ.lowerBound) + Self.lowercaseAZ.lowerBound : self + } + + var uppercasedUTF8Byte: UInt8 { + return isLowercase ? (self - Self.lowercaseAZ.lowerBound) + Self.uppercaseAZ.lowerBound : self + } + + var isDelimiter: Bool { + return (self == .cSpace) + || (self == .cPlus) + || (self == .cMinus) + || (self == .cColon) + || (self == .cPeriod) + || (self == .cUnderscore) + || (self == .cLeftParentheses) + || (self == .cRightParentheses) + } +} diff --git a/Sources/CompletionScoring/Utilities/SelectTopK.swift b/Sources/CompletionScoring/Utilities/SelectTopK.swift new file mode 100644 index 000000000..cfa6929d0 --- /dev/null +++ b/Sources/CompletionScoring/Utilities/SelectTopK.swift @@ -0,0 +1,134 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +extension UnsafeMutableBufferPointer { + /// Find the top `k` elements as ordered by `lessThan`, and move them to the front of the receiver. + /// The order of elements at positions `k.. Bool) { + if k < count { + withoutActuallyEscaping(lessThan) { lessThan in + var sorter = HeapSorter(self, orderedBy: lessThan) + sorter.sortToBack(maxSteps: k) + let slide = count - k + for frontIdx in 0.. { + var heapStorage: UnsafeMutablePointer + var heapCount: Int + var inOrder: (Element, Element) -> Bool + + init(_ storage: UnsafeMutableBufferPointer, orderedBy lessThan: @escaping (Element, Element) -> Bool) { + self.heapCount = storage.count + self.heapStorage = storage.baseAddress! + + self.inOrder = { lhs, rhs in // Make a `<=` out of `<`, we don't need to push down when equal + return lessThan(lhs, rhs) || !lessThan(rhs, lhs) + } + if heapCount > 0 { + let lastIndex = heapLastIndex + let lastItemOnSecondLevelFromBottom = lastIndex.parent + for index in (0...lastItemOnSecondLevelFromBottom.value).reversed() { + pushParentDownIfNeeded(at: HeapIndex(index)) + } + } + } + + struct HeapIndex: Comparable { + var value: Int + + init(_ value: Int) { + self.value = value + } + + var parent: Self { .init((value - 1) / 2) } + var leftChild: Self { .init((value * 2) + 1) } + var rightChild: Self { .init((value * 2) + 2) } + + static func < (lhs: Self, rhs: Self) -> Bool { + lhs.value < rhs.value + } + } + + var heapStartIndex: HeapIndex { .init(0) } + var heapLastIndex: HeapIndex { .init(heapCount - 1) } + var heapEndIndex: HeapIndex { .init(heapCount) } + + subscript(heapIndex heapIndex: HeapIndex) -> Element { + get { + heapStorage[heapIndex.value] + } + set { + heapStorage[heapIndex.value] = newValue + } + } + + func heapSwap(_ a: HeapIndex, _ b: HeapIndex) { + let t = heapStorage[a.value] + heapStorage[a.value] = heapStorage[b.value] + heapStorage[b.value] = t + } + + mutating func sortToBack(maxSteps: Int) { + precondition(maxSteps < heapCount) + for _ in 0.. 0 { + try check(heapStartIndex) + } + } +} diff --git a/Sources/CompletionScoring/Utilities/Serialization/BinaryCodable.swift b/Sources/CompletionScoring/Utilities/Serialization/BinaryCodable.swift new file mode 100644 index 000000000..12329979d --- /dev/null +++ b/Sources/CompletionScoring/Utilities/Serialization/BinaryCodable.swift @@ -0,0 +1,66 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +/// BinaryCodable, along with BinaryEncoder, and BinaryDecoder enable streaming values into a byte array representation. +/// To support BinaryCodable, implement the encode method, and write all of your fields into the coder using +/// `encoder.write()`. In your `init(BinaryDecoder)` method, decode those same fields using `init(BinaryDecoder)`. +/// +/// A typical conformance is as simple as: +/// ``` +/// struct Size: BinaryCodable { +/// var width: Double +/// var height: Double +/// +/// func encode(_ encoder: inout BinaryEncoder) { +/// encoder.write(width) +/// encoder.write(height) +/// } +/// +/// init(_ decoder: inout BinaryDecoder) throws { +/// width = try Double(&decoder) +/// height = try Double(&decoder) +/// } +/// } +/// ``` +/// +/// The encoding is very minimal. There is no metadata in the stream, and decode purely has meaning based on what order +/// clients decode values, and which types they use. If your encoder encodes a bool and two ints, your decoder must +/// decode a bool and two ints, otherwise the next structure to be decoded would read what ever you didn't decode, +/// rather than what it encoded. +package protocol BinaryCodable { + + /// Initialize self using values previously writen in `encode(_:)`. All values written by `encode(_:)` must be read + /// by `init(_:)`, in the same order, using the same types. Otherwise the next structure to decode will read the + /// last value you didn't read rather than the first value it wrote. + init(_ decoder: inout BinaryDecoder) throws + + /// Recursively encode content using `encoder.write(_:)` + func encode(_ encoder: inout BinaryEncoder) +} + +extension BinaryCodable { + /// Convenience method to encode a structure to a byte array + package func binaryCodedRepresentation(contentVersion: Int) -> [UInt8] { + BinaryEncoder.encode(contentVersion: contentVersion) { encoder in + encoder.write(self) + } + } + + /// Convenience method to decode a structure from a byte array + package init(binaryCodedRepresentation: [UInt8]) throws { + self = try BinaryDecoder.decode(bytes: binaryCodedRepresentation) { decoder in + try Self(&decoder) + } + } +} diff --git a/Sources/CompletionScoring/Utilities/Serialization/BinaryDecoder.swift b/Sources/CompletionScoring/Utilities/Serialization/BinaryDecoder.swift new file mode 100644 index 000000000..c68524394 --- /dev/null +++ b/Sources/CompletionScoring/Utilities/Serialization/BinaryDecoder.swift @@ -0,0 +1,92 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +package struct BinaryDecoder { + private let stream: [UInt8] + private var position = 0 + + static let maximumUnderstoodStreamVersion = BinaryEncoder.maximumUnderstoodStreamVersion + private(set) var contentVersion: Int + + private init(stream: [UInt8]) throws { + self.stream = stream + self.contentVersion = 0 + + let streamVersion = try Int(&self) + if streamVersion > Self.maximumUnderstoodStreamVersion { + throw GenericError("Stream version is too new: \(streamVersion)") + } + self.contentVersion = try Int(&self) + } + + /// Top level function to begin decoding. + /// - Parameters: + /// - body: a closure accepting a `BinaryDecoder` that you can make `init(_:)` calls against to decode the + /// archive. + /// - Returns: The value (if any) returned by the body block. + static func decode(bytes: [UInt8], _ body: (inout Self) throws -> R) throws -> R { + var decoder = try BinaryDecoder(stream: bytes) + let decoded = try body(&decoder) + if decoder.position != decoder.stream.count { + // 99% of the time, the client didn't line up their reads and writes, and just decoded garbage. It's more important to catch this than to allow it for some hypothetical use case. + throw GenericError("Unaligned decode") + } + return decoded + } + + private var bytesRemaining: Int { + stream.count - position + } + + // Return the next `byteCount` bytes from the archvie, and advance the read location. + // Throws if there aren't enough bytes in the archive. + mutating func readRawBytes(count byteCount: Int) throws -> ArraySlice { + if bytesRemaining >= byteCount && byteCount >= 0 { + let slice = stream[position ..+ byteCount] + position += byteCount + return slice + } else { + throw GenericError("Stream has \(bytesRemaining) bytes renamining, requires \(byteCount)") + } + } + + // Return the next byte from the archvie, and advance the read location. Throws if there aren't any more bytes in + // the archive. + mutating func readByte() throws -> UInt8 { + let slice = try readRawBytes(count: 1) + return slice[slice.startIndex] + } + + // Read the next bytes from the archive into the memory holding `V`. Useful for decoding primitive values like + // `UInt32`. All architecture specific constraints like endianness, or sizing, are the responsibility of the caller. + mutating func read(rawBytesInto result: inout V) throws { + try withUnsafeMutableBytes(of: &result) { valueBytes in + let slice = try readRawBytes(count: valueBytes.count) + for (offset, byte) in slice.enumerated() { + valueBytes[offset] = byte + } + } + } + + // A convenience method for decoding an enum, and throwing a common error for an unknown case. The body block can + // decode additional additional payload for data associated with each enum case. + mutating func decodeEnumByte(body: (inout BinaryDecoder, UInt8) throws -> E?) throws -> E { + let numericRepresentation = try readByte() + if let decoded = try body(&self, numericRepresentation) { + return decoded + } else { + throw GenericError("Invalid encoding of \(E.self): \(numericRepresentation)") + } + } +} diff --git a/Sources/CompletionScoring/Utilities/Serialization/BinaryEncoder.swift b/Sources/CompletionScoring/Utilities/Serialization/BinaryEncoder.swift new file mode 100644 index 000000000..81f37c32e --- /dev/null +++ b/Sources/CompletionScoring/Utilities/Serialization/BinaryEncoder.swift @@ -0,0 +1,59 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +package struct BinaryEncoder { + private var stream: [UInt8] = [] + static let maximumUnderstoodStreamVersion = 0 + let contentVersion: Int + + private init(contentVersion: Int) { + self.contentVersion = contentVersion + write(Self.maximumUnderstoodStreamVersion) + write(contentVersion) + } + + /// Top level function to begin encoding. + /// - Parameters: + /// - contentVersion: A version number for the content of the whole archive. + /// - body: a closure accepting a `BinaryEncoder` that you can make `write(_:)` calls against to populate the + /// archive. + /// - Returns: a byte array that can be used with `BinaryDecoder` + static func encode(contentVersion: Int, _ body: (inout Self) -> ()) -> [UInt8] { + var encoder = BinaryEncoder(contentVersion: contentVersion) + body(&encoder) + return encoder.stream + } + + /// Write the literal bytes of `value` into the archive. The client is responsible for any endian or architecture + /// sizing considerations. + mutating func write(rawBytesOf value: V) { + withUnsafeBytes(of: value) { valueBytes in + write(rawBytes: valueBytes) + } + } + + /// Write `rawBytes` into the archive. You might use this to encode the contents of a bitmap, or a UTF8 sequence. + mutating func write(rawBytes: C) where C.Element == UInt8 { + stream.append(contentsOf: rawBytes) + } + + mutating func writeByte(_ value: UInt8) { + write(value) + } + + /// Recursively encode `value` and all of it's contents. + mutating func write(_ value: V) { + value.encode(&self) + } +} diff --git a/Sources/CompletionScoring/Utilities/Serialization/Conformances/Array+BinaryCodable.swift b/Sources/CompletionScoring/Utilities/Serialization/Conformances/Array+BinaryCodable.swift new file mode 100644 index 000000000..966ee2d7d --- /dev/null +++ b/Sources/CompletionScoring/Utilities/Serialization/Conformances/Array+BinaryCodable.swift @@ -0,0 +1,30 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +extension Array: BinaryCodable where Element: BinaryCodable { + package init(_ decoder: inout BinaryDecoder) throws { + let count = try Int(&decoder) + self.init(capacity: count) + for _ in 0..() -> [T] where Element == T? { + compactMap { $0 } + } +} + +extension Sequence where Element: Numeric { + package func sum() -> Element { + reduce(0, +) + } +} + +extension Sequence { + func sum(of valueExtractor: (Element) -> I) -> I { + reduce(into: 0) { partialResult, element in + partialResult += valueExtractor(element) + } + } +} + +extension Sequence { + func countOf(predicate: (Element) throws -> Bool) rethrows -> Int { + var count = 0 + for element in self { + if try predicate(element) { + count += 1 + } + } + return count + } +} + +struct GenericError: Error, LocalizedError { + var message: String + init(_ message: String) { + self.message = message + } +} + +extension Optional { + package func unwrap(orThrow message: String) throws -> Wrapped { + if let result = self { + return result + } + throw GenericError(message) + } + + package func unwrap(orFail message: String) -> Wrapped { + if let result = self { + return result + } + preconditionFailure(message) + } + + package mutating func lazyInitialize(initializer: () -> Wrapped) -> Wrapped { + if let wrapped = self { + return wrapped + } else { + let wrapped = initializer() + self = wrapped + return wrapped + } + } +} + +extension UnsafeBufferPointer { + init(to element: inout Element) { + self = withUnsafePointer(to: &element) { pointer in + UnsafeBufferPointer(start: pointer, count: 1) + } + } +} + +extension UnsafeBufferPointer { + package static func allocate(copyOf original: some Collection) -> Self { + return Self(UnsafeMutableBufferPointer.allocate(copyOf: original)) + } +} + +extension UnsafeMutablePointer { + mutating func resize(fromCount oldCount: Int, toCount newCount: Int) { + let replacement = UnsafeMutablePointer.allocate(capacity: newCount) + let copiedCount = min(oldCount, newCount) + replacement.moveInitialize(from: self, count: copiedCount) + let abandondedCount = oldCount - copiedCount + self.advanced(by: copiedCount).deinitialize(count: abandondedCount) + deallocate() + self = replacement + } + + func initialize(from collection: some Collection) { + let buffer = UnsafeMutableBufferPointer(start: self, count: collection.count) + _ = buffer.initialize(from: collection) + } +} + +extension UnsafeMutableBufferPointer { + package static func allocate(copyOf original: some Collection) -> Self { + let copy = UnsafeMutableBufferPointer.allocate(capacity: original.count) + _ = copy.initialize(from: original) + return copy + } + + package func initialize(index: Int, to value: Element) { + self.baseAddress!.advanced(by: index).initialize(to: value) + } + + func deinitializeAll() { + baseAddress!.deinitialize(count: count) + } + + package func deinitializeAllAndDeallocate() { + deinitializeAll() + deallocate() + } + + mutating func truncateAndDeinitializeTail(maxLength: Int) { + if maxLength < count { + self.baseAddress!.advanced(by: maxLength).deinitialize(count: count - maxLength) + self = UnsafeMutableBufferPointer(start: baseAddress, count: maxLength) + } + } + + mutating func removeAndTruncateWhere(_ predicate: (Element) -> Bool) { + var writeIndex = 0 + for readIndex in indices { + if !predicate(self[readIndex]) { + if writeIndex != readIndex { + swapAt(writeIndex, readIndex) + } + writeIndex += 1 + } + } + truncateAndDeinitializeTail(maxLength: writeIndex) + } + + func setAll(to value: Element) { + for index in indices { + self[index] = value + } + } +} + +infix operator ? : ComparisonPrecedence + +extension Comparable { + /// Useful for chained comparison, for example on a person, sorting by last, first, age: + /// ``` + /// static func <(_ lhs: Self, _ rhs: Self) -> Bool { + /// return lhs.last Bool? { + // Assume that `<` is most likely, and avoid a redundant `==`. + if lhs < rhs { + return true + } else if lhs == rhs { + return nil + } else { + return false + } + } + + /// See ? (_ lhs: Self, _ rhs: Self) -> Bool? { + // Assume that `>` is most likely, and avoid a redundant `==`. + if lhs > rhs { + return true + } else if lhs == rhs { + return nil + } else { + return false + } + } +} + +infix operator ..+ : RangeFormationPrecedence + +package func ..+ (lhs: Bound, rhs: Bound) -> Range { + lhs..<(lhs + rhs) +} + +extension RandomAccessCollection { + fileprivate func withMapScratchArea(body: (UnsafeMutablePointer) -> R) -> R { + let scratchArea = UnsafeMutablePointer.allocate(capacity: count) + defer { + scratchArea.deinitialize(count: count) + /// Should be a no-op + scratchArea.deallocate() + } + return body(scratchArea) + } + + package func concurrentCompactMap(_ f: @Sendable (Element) -> T?) -> [T] where Self: Sendable { + return withMapScratchArea { (results: UnsafeMutablePointer) -> [T] in + // `nonisolated(unsafe)` is fine because we write to different offsets within the buffer on every concurrent + // iteration. + nonisolated(unsafe) let results = results + DispatchQueue.concurrentPerform(iterations: count) { iterationIndex in + let collectionIndex = self.index(self.startIndex, offsetBy: iterationIndex) + results.advanced(by: iterationIndex).initialize(to: f(self[collectionIndex])) + } + return UnsafeBufferPointer(start: results, count: count).compacted() + } + } + + package func concurrentMap(_ f: @Sendable (Element) -> T) -> [T] where Self: Sendable { + return withMapScratchArea { (results: UnsafeMutablePointer) -> [T] in + // `nonisolated(unsafe)` is fine because we write to different offsets within the buffer on every concurrent + // iteration. + nonisolated(unsafe) let results = results + DispatchQueue.concurrentPerform(iterations: count) { iterationIndex in + let collectionIndex = self.index(self.startIndex, offsetBy: iterationIndex) + results.advanced(by: iterationIndex).initialize(to: f(self[collectionIndex])) + } + return Array(UnsafeBufferPointer(start: results, count: count)) + } + } + + package func max(by accessor: (Element) -> some Comparable) -> Element? { + self.max { lhs, rhs in + accessor(lhs) < accessor(rhs) + } + } + + package func min(by accessor: (Element) -> some Comparable) -> Element? { + self.min { lhs, rhs in + accessor(lhs) < accessor(rhs) + } + } + + package func max(of accessor: (Element) -> T) -> T? { + let extreme = self.max { lhs, rhs in + accessor(lhs) < accessor(rhs) + } + return extreme.map(accessor) + } + + package func min(of accessor: (Element) -> T) -> T? { + let extreme = self.min { lhs, rhs in + accessor(lhs) < accessor(rhs) + } + return extreme.map(accessor) + } +} + +protocol ContiguousZeroBasedIndexedCollection: Collection where Index == Int { + var indices: Range { get } +} + +extension ContiguousZeroBasedIndexedCollection { + func slicedConcurrentForEachSliceRange(body: @Sendable (Range) -> ()) { + // We want to use `DispatchQueue.concurrentPerform`, but we want to be called only a few times. So that we + // can amortize per-callback work. We also want to oversubscribe so that we can efficiently use + // heterogeneous CPUs. If we had 4 efficiency cores, and 4 performance cores, and we dispatched 8 work items + // we'd finish 4 quickly, and then either migrate the work, or leave the performance cores idle. Scheduling + // extra jobs should let the performance cores pull a disproportionate amount of work items. More fine + // granularity also helps if the work items aren't all the same difficulty, for the same reason. + + // Defensive against `processorCount` failing + let sliceCount = Swift.min(Swift.max(ProcessInfo.processInfo.processorCount * 32, 1), count) + let count = self.count + DispatchQueue.concurrentPerform(iterations: sliceCount) { sliceIndex in + precondition(sliceCount >= 1) + /// Remainder will be distributed across leading slices, so slicing an array with count 5 into 3 slices will give you + /// slices of size [2, 2, 1]. + let equalPortion = count / sliceCount + let remainder = count - (equalPortion * sliceCount) + let getsRemainder = sliceIndex < remainder + let length = equalPortion + (getsRemainder ? 1 : 0) + let previousSlicesGettingRemainder = Swift.min(sliceIndex, remainder) + let start = (sliceIndex * equalPortion) + previousSlicesGettingRemainder + body(start ..+ length) + } + } +} + +extension Array: ContiguousZeroBasedIndexedCollection {} +extension UnsafeMutableBufferPointer: ContiguousZeroBasedIndexedCollection {} + +extension Array { + /// A concurrent map that allows amortizing per-thread work. For example, if you need a scratch buffer + /// to complete the mapping, but could use the same scratch buffer for every iteration on the same thread + /// you can use this function instead of `concurrentMap`. This method also often helps amortize reference + /// counting since there are less callbacks + /// + /// - Important: The callback must write to all values in `destination[0..( + writer: @Sendable (ArraySlice, _ destination: UnsafeMutablePointer) -> () + ) -> [T] where Self: Sendable { + return Array(unsafeUninitializedCapacity: count) { buffer, initializedCount in + if let bufferBase = buffer.baseAddress { + // `nonisolated(unsafe)` is fine because every concurrent iteration accesses a disjunct slice of `buffer`. + nonisolated(unsafe) let bufferBase = bufferBase + slicedConcurrentForEachSliceRange { sliceRange in + writer(self[sliceRange], bufferBase.advanced(by: sliceRange.startIndex)) + } + } else { + precondition(isEmpty) + } + initializedCount = count + } + } + + /// Concurrent for-each on self, but slice based to allow the body to amortize work across callbacks + func slicedConcurrentForEach(body: @Sendable (ArraySlice) -> ()) where Self: Sendable { + slicedConcurrentForEachSliceRange { sliceRange in + body(self[sliceRange]) + } + } + + func concurrentForEach(body: @Sendable (Element) -> ()) where Self: Sendable { + DispatchQueue.concurrentPerform(iterations: count) { index in + body(self[index]) + } + } + + init(capacity: Int) { + self = Self() + reserveCapacity(capacity) + } + + package init(count: Int, generator: () -> Element) { + self = (0..(overwritingDuplicates: Affirmative, _ map: (Key) -> K) -> [K: Value] { + var result = Dictionary(capacity: count) + for (key, value) in self { + result[map(key)] = value + } + return result + } +} + +enum Affirmative { + case affirmative +} + +package enum ComparisonOrder: Equatable { + case ascending + case same + case descending + + init(_ value: Int) { + if value < 0 { + self = .ascending + } else if value == 0 { + self = .same + } else { + self = .descending + } + } +} + +extension UnsafeBufferPointer { + func afterFirst() -> Self { + precondition(hasContent) + return UnsafeBufferPointer(start: baseAddress! + 1, count: count - 1) + } + + package static func withSingleElementBuffer( + of element: Element, + body: (Self) throws -> R + ) rethrows -> R { + var element = element + let typedBufferPointer = Self(to: &element) + return try body(typedBufferPointer) + } +} + +extension UnsafeBufferPointer { + package func rangeOf(bytes needle: UnsafeBufferPointer, startOffset: Int = 0) -> Range? { + baseAddress.flatMap { baseAddress in + let match = memmem(baseAddress + startOffset, count - startOffset, needle.baseAddress, needle.count) + return match.map { match in + let start = baseAddress.distance(to: match.assumingMemoryBound(to: UInt8.self)) + return start ..+ needle.count + } + } + } + + func rangeOf(bytes needle: [UInt8]) -> Range? { + needle.withUnsafeBufferPointer { bytes in + rangeOf(bytes: bytes) + } + } +} + +func equateBytes(_ lhs: UnsafeBufferPointer, _ rhs: UnsafeBufferPointer) -> Bool { + compareBytes(lhs, rhs) == .same +} + +package func compareBytes( + _ lhs: UnsafeBufferPointer, + _ rhs: UnsafeBufferPointer +) -> ComparisonOrder { + compareBytes(UnsafeRawBufferPointer(lhs), UnsafeRawBufferPointer(rhs)) +} + +func compareBytes(_ lhs: UnsafeRawBufferPointer, _ rhs: UnsafeRawBufferPointer) -> ComparisonOrder { + let result = Int(memcmp(lhs.baseAddress!, rhs.baseAddress!, min(lhs.count, rhs.count))) + return (result != 0) ? ComparisonOrder(result) : ComparisonOrder(lhs.count - rhs.count) +} + +extension String { + /// Non mutating version of withUTF8. withUTF8 is mutating to make the string contiguous, so that future calls will + /// be cheaper. + /// Useful when you're operating on an argument, and have no way to avoid this copy dance. + package func withUncachedUTF8Bytes( + _ body: ( + UnsafeBufferPointer + ) throws -> R + ) rethrows -> R { + var copy = self + return try copy.withUTF8(body) + } +} diff --git a/Sources/CompletionScoring/Utilities/UnsafeArray.swift b/Sources/CompletionScoring/Utilities/UnsafeArray.swift new file mode 100644 index 000000000..17bf3f290 --- /dev/null +++ b/Sources/CompletionScoring/Utilities/UnsafeArray.swift @@ -0,0 +1,91 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +/// A manually allocated and deallocated array with automatic growth on insert. +/// +/// - Warning: this type is Unsafe. Writing to an allocated instance must be exclusive to one client. +/// Multiple readers are OK, as long as deallocation is coordinated. Writing must be exclusive because +/// appends can cause realloc, leaving dangling pointers in the copies held by other clients. Appends +/// also would not update the counts in other clients. +internal struct UnsafeArray { + private(set) var count = 0 + private(set) var capacity: Int + private(set) var elements: UnsafeMutablePointer + + private init(elements: UnsafeMutablePointer, capacity: Int) { + self.capacity = capacity + self.elements = elements + } + + /// Must be deallocated with `deallocate()`. Will grow beyond `initialCapacity` as elements are added. + static func allocate(initialCapacity: Int) -> Self { + Self(elements: UnsafeMutablePointer.allocate(capacity: initialCapacity), capacity: initialCapacity) + } + + mutating func deallocate() { + elements.deinitialize(count: count) + elements.deallocate() + count = 0 + capacity = 0 + } + + /// Must be deallocated with `deallocate()`. + func allocateCopy(preservingCapacity: Bool) -> Self { + var copy = UnsafeArray.allocate(initialCapacity: preservingCapacity ? capacity : count) + copy.elements.initialize(from: elements, count: count) + copy.count = count + return copy + } + + private mutating func resize(newCapacity: Int) { + assert(newCapacity >= count) + elements.resize(fromCount: count, toCount: newCapacity) + capacity = newCapacity + } + + mutating func reserve(minimumAdditionalCapacity: Int) { + let availableAdditionalCapacity = (capacity - count) + if availableAdditionalCapacity < minimumAdditionalCapacity { + resize(newCapacity: max(capacity * 2, capacity + minimumAdditionalCapacity)) + } + } + + mutating func append(_ element: Element) { + reserve(minimumAdditionalCapacity: 1) + elements[count] = element + count += 1 + } + + mutating func append(contentsOf collection: some Collection) { + reserve(minimumAdditionalCapacity: collection.count) + elements.advanced(by: count).initialize(from: collection) + count += collection.count + } + + private func assertBounds(_ index: Int) { + assert(index >= 0) + assert(index < count) + } + + subscript(_ index: Int) -> Element { + get { + assertBounds(index) + return elements[index] + } + set { + assertBounds(index) + elements[index] = newValue + } + } +} diff --git a/Sources/CompletionScoring/Utilities/UnsafeStackAllocator.swift b/Sources/CompletionScoring/Utilities/UnsafeStackAllocator.swift new file mode 100644 index 000000000..740327aab --- /dev/null +++ b/Sources/CompletionScoring/Utilities/UnsafeStackAllocator.swift @@ -0,0 +1,245 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +private struct Bytes8 { var storage: (UInt64) = (0) } +private struct Bytes16 { var storage: (Bytes8, Bytes8) = (.init(), .init()) } +private struct Bytes32 { var storage: (Bytes16, Bytes16) = (.init(), .init()) } +private struct Bytes64 { var storage: (Bytes32, Bytes32) = (.init(), .init()) } +private struct Bytes128 { var storage: (Bytes64, Bytes64) = (.init(), .init()) } +private struct Bytes256 { var storage: (Bytes128, Bytes128) = (.init(), .init()) } +private struct Bytes512 { var storage: (Bytes256, Bytes256) = (.init(), .init()) } +private struct Bytes1024 { var storage: (Bytes512, Bytes512) = (.init(), .init()) } +private struct Bytes2048 { var storage: (Bytes1024, Bytes1024) = (.init(), .init()) } +private struct Bytes4096 { var storage: (Bytes2048, Bytes2048) = (.init(), .init()) } +private struct Bytes8192 { var storage: (Bytes4096, Bytes4096) = (.init(), .init()) } + +package struct UnsafeStackAllocator { + private typealias Storage = Bytes8192 + private var storage = Storage() + private static let pageSize = 64 + private static let storageCapacity = MemoryLayout.size + private var pagesAllocated = 0 + private var pagesAvailable: Int { + pagesCapacity - pagesAllocated + } + + private init() { + } + + package static func withUnsafeStackAllocator(body: (inout Self) throws -> R) rethrows -> R { + var allocator = Self() + defer { assert(allocator.pagesAllocated == 0) } + return try body(&allocator) + } + + private let pagesCapacity = Self.storageCapacity / Self.pageSize + + private func pages(for type: Element.Type, maximumCapacity: Int) -> Int { + let bytesNeeded = MemoryLayout.stride * maximumCapacity + return ((bytesNeeded - 1) / Self.pageSize) + 1 + } + + private mutating func allocate( + of type: Element.Type, + maximumCapacity: Int + ) -> UnsafeMutablePointer { + // Avoid dealing with alignment for now. + assert(MemoryLayout.alignment <= MemoryLayout.alignment) + // Avoid dealing with alignment for now. + assert(MemoryLayout.alignment <= Self.pageSize) + let pagesNeeded = pages(for: type, maximumCapacity: maximumCapacity) + if pagesNeeded < pagesAvailable { + return withUnsafeMutableBytes(of: &storage) { arena in + let start = arena.baseAddress!.advanced(by: pagesAllocated * Self.pageSize).bindMemory( + to: Element.self, + capacity: maximumCapacity + ) + pagesAllocated += pagesNeeded + return start + } + } else { + return UnsafeMutablePointer.allocate(capacity: maximumCapacity) + } + } + + mutating func allocateBuffer(of type: Element.Type, count: Int) -> UnsafeMutableBufferPointer { + return UnsafeMutableBufferPointer(start: allocate(of: type, maximumCapacity: count), count: count) + } + + mutating func allocateUnsafeArray( + of type: Element.Type, + maximumCapacity: Int + ) -> UnsafeStackArray { + UnsafeStackArray(base: allocate(of: type, maximumCapacity: maximumCapacity), capacity: maximumCapacity) + } + + mutating private func deallocate(_ base: UnsafePointer, capacity: Int) { + let arrayStart = UnsafeRawPointer(base) + let arrayPages = pages(for: Element.self, maximumCapacity: capacity) + withUnsafeBytes(of: &storage) { arena in + let arenaStart = UnsafeRawPointer(arena.baseAddress)! + let arenaEnd = arenaStart.advanced(by: Self.storageCapacity) + if (arrayStart >= arenaStart) && (arrayStart < arenaEnd) { + let projectedArrayStart = arenaStart.advanced(by: (pagesAllocated - arrayPages) * Self.pageSize) + assert(projectedArrayStart == arrayStart, "deallocate(...) must be called in FIFO order.") + pagesAllocated -= arrayPages + } else { + arrayStart.deallocate() + } + } + } + + /// - Note: `buffer.count` must the be the same as from the original allocation. + /// - Note: deiniting buffer contents is caller's responsibility. + mutating func deallocate(_ buffer: inout UnsafeBufferPointer) { + if let baseAddress = buffer.baseAddress { + deallocate(baseAddress, capacity: buffer.count) + buffer = UnsafeBufferPointer(start: nil, count: 0) + } + } + + /// - Note: `buffer.count` must the be the same as from the original allocation. + /// - Note: deiniting buffer contents is caller's responsibility. + mutating func deallocate(_ buffer: inout UnsafeMutableBufferPointer) { + if let baseAddress = buffer.baseAddress { + deallocate(baseAddress, capacity: buffer.count) + buffer = UnsafeMutableBufferPointer(start: nil, count: 0) + } + } + + mutating func deallocate(_ array: inout UnsafeStackArray) { + array.prepareToDeallocate() + deallocate(array.base, capacity: array.capacity) + array = UnsafeStackArray(base: array.base, capacity: 0) + } + + package mutating func withStackArray( + of elementType: Element.Type, + maximumCapacity: Int, + body: (inout UnsafeStackArray) throws -> R + ) rethrows -> R { + var stackArray = allocateUnsafeArray(of: elementType, maximumCapacity: maximumCapacity); + defer { deallocate(&stackArray) } + return try body(&stackArray) + } +} + +package struct UnsafeStackArray { + private(set) var base: UnsafeMutablePointer + fileprivate let capacity: Int + package private(set) var count = 0 + + fileprivate init(base: UnsafeMutablePointer, capacity: Int) { + self.base = base + self.capacity = capacity + } + + fileprivate mutating func prepareToDeallocate() { + removeAll() // Contained elements may need de-init + } + + // Assume the memory is initialized with whatever is there. Only safe on trivial types. + mutating func initializeWithContainedGarbage() { + count = capacity + } + + package mutating func fill(with element: Element) { + while count < capacity { + append(element) + } + } + + package mutating func removeAll() { + base.deinitialize(count: count) + count = 0 + } + + mutating func append(contentsOf sequence: some Sequence) { + for element in sequence { + append(element) + } + } + + package mutating func append(_ element: Element) { + assert(count < capacity) + (base + count).initialize(to: element) + count += 1 + } + + mutating func push(_ element: Element) { + append(element) + } + + package mutating func removeLast() { + assert(count > 0) + (base + count - 1).deinitialize(count: 1) + count -= 1 + } + + package subscript(_ index: Int) -> Element { + get { + assert(index < count) + return (base + index).pointee + } + set { + assert(index < count) + (base + index).pointee = newValue + } + } + + package mutating func truncate(to countLimit: Int) { + assert(countLimit >= 0) + if count > countLimit { + (base + countLimit).deinitialize(count: count - countLimit) + count = countLimit + } + } + + mutating func truncateLeavingGarbage(to countLimit: Int) { + assert(countLimit >= 0) + count = countLimit + } + + var contiguousStorage: UnsafeBufferPointer { + UnsafeBufferPointer(start: base, count: count) + } + + func contiguousStorage(count viewCount: Int) -> UnsafeBufferPointer { + assert(viewCount <= count) + return UnsafeBufferPointer(start: base, count: viewCount) + } +} + +extension UnsafeStackArray: RandomAccessCollection { + package var startIndex: Int { + 0 + } + + package var endIndex: Int { + count + } +} + +extension UnsafeStackArray: MutableCollection { +} + +extension UnsafeStackArray { + package mutating func popLast() -> Element? { + let last = last + if hasContent { + removeLast() + } + return last + } +} diff --git a/Sources/CompletionScoringForPlugin b/Sources/CompletionScoringForPlugin new file mode 120000 index 000000000..097f08392 --- /dev/null +++ b/Sources/CompletionScoringForPlugin @@ -0,0 +1 @@ +CompletionScoring/ \ No newline at end of file diff --git a/Sources/CompletionScoringTestSupport/CommonFunctionTerms.json b/Sources/CompletionScoringTestSupport/CommonFunctionTerms.json new file mode 100644 index 000000000..88ec39d2d --- /dev/null +++ b/Sources/CompletionScoringTestSupport/CommonFunctionTerms.json @@ -0,0 +1,994 @@ +[ +{ "word" : "Should", "count" : 14}, +{ "word" : "Suggested", "count" : 14}, +{ "word" : "Trigger", "count" : 14}, +{ "word" : "any", "count" : 14}, +{ "word" : "calculate", "count" : 14}, +{ "word" : "constructor", "count" : 14}, +{ "word" : "direct", "count" : 14}, +{ "word" : "dummy", "count" : 14}, +{ "word" : "empty", "count" : 14}, +{ "word" : "error", "count" : 14}, +{ "word" : "last", "count" : 14}, +{ "word" : "length", "count" : 14}, +{ "word" : "metadata", "count" : 14}, +{ "word" : "obj", "count" : 14}, +{ "word" : "objcpp", "count" : 14}, +{ "word" : "pasteboard", "count" : 14}, +{ "word" : "rename", "count" : 14}, +{ "word" : "save", "count" : 14}, +{ "word" : "selector", "count" : 14}, +{ "word" : "service", "count" : 14}, +{ "word" : "title", "count" : 14}, +{ "word" : "when", "count" : 14}, +{ "word" : "Body", "count" : 15}, +{ "word" : "CALayer", "count" : 15}, +{ "word" : "CGColor", "count" : 15}, +{ "word" : "Cases", "count" : 15}, +{ "word" : "Check", "count" : 15}, +{ "word" : "Clicked", "count" : 15}, +{ "word" : "Coordinator", "count" : 15}, +{ "word" : "Date", "count" : 15}, +{ "word" : "Deleted", "count" : 15}, +{ "word" : "Dict", "count" : 15}, +{ "word" : "Heading", "count" : 15}, +{ "word" : "Message", "count" : 15}, +{ "word" : "Metadata", "count" : 15}, +{ "word" : "More", "count" : 15}, +{ "word" : "NSImage", "count" : 15}, +{ "word" : "NSPoint", "count" : 15}, +{ "word" : "NSValidated", "count" : 15}, +{ "word" : "Parent", "count" : 15}, +{ "word" : "Popover", "count" : 15}, +{ "word" : "Received", "count" : 15}, +{ "word" : "Sequence", "count" : 15}, +{ "word" : "UIContext", "count" : 15}, +{ "word" : "UIEvent", "count" : 15}, +{ "word" : "active", "count" : 15}, +{ "word" : "async", "count" : 15}, +{ "word" : "closest", "count" : 15}, +{ "word" : "disable", "count" : 15}, +{ "word" : "extension", "count" : 15}, +{ "word" : "framework", "count" : 15}, +{ "word" : "nif", "count" : 15}, +{ "word" : "parameters", "count" : 15}, +{ "word" : "prepare", "count" : 15}, +{ "word" : "previous", "count" : 15}, +{ "word" : "run", "count" : 15}, +{ "word" : "sorted", "count" : 15}, +{ "word" : "specification", "count" : 15}, +{ "word" : "suggestion", "count" : 15}, +{ "word" : "top", "count" : 15}, +{ "word" : "unregister", "count" : 15}, +{ "word" : "Activity", "count" : 16}, +{ "word" : "Balance", "count" : 16}, +{ "word" : "Bounds", "count" : 16}, +{ "word" : "Byte", "count" : 16}, +{ "word" : "Child", "count" : 16}, +{ "word" : "Converter", "count" : 16}, +{ "word" : "DVTFile", "count" : 16}, +{ "word" : "Dynamic", "count" : 16}, +{ "word" : "Indentation", "count" : 16}, +{ "word" : "Jump", "count" : 16}, +{ "word" : "Local", "count" : 16}, +{ "word" : "NSColor", "count" : 16}, +{ "word" : "Notifications", "count" : 16}, +{ "word" : "Operations", "count" : 16}, +{ "word" : "Out", "count" : 16}, +{ "word" : "Register", "count" : 16}, +{ "word" : "Replacing", "count" : 16}, +{ "word" : "Selecting", "count" : 16}, +{ "word" : "Shown", "count" : 16}, +{ "word" : "Storage", "count" : 16}, +{ "word" : "Strategy", "count" : 16}, +{ "word" : "Struct", "count" : 16}, +{ "word" : "That", "count" : 16}, +{ "word" : "UIScene", "count" : 16}, +{ "word" : "Updates", "count" : 16}, +{ "word" : "Values", "count" : 16}, +{ "word" : "Without", "count" : 16}, +{ "word" : "args", "count" : 16}, +{ "word" : "attributes", "count" : 16}, +{ "word" : "children", "count" : 16}, +{ "word" : "contents", "count" : 16}, +{ "word" : "decode", "count" : 16}, +{ "word" : "dismiss", "count" : 16}, +{ "word" : "dynamic", "count" : 16}, +{ "word" : "external", "count" : 16}, +{ "word" : "extract", "count" : 16}, +{ "word" : "generate", "count" : 16}, +{ "word" : "inner", "count" : 16}, +{ "word" : "local", "count" : 16}, +{ "word" : "minimap", "count" : 16}, +{ "word" : "original", "count" : 16}, +{ "word" : "protocol", "count" : 16}, +{ "word" : "relative", "count" : 16}, +{ "word" : "sdk", "count" : 16}, +{ "word" : "Auto", "count" : 17}, +{ "word" : "Closing", "count" : 17}, +{ "word" : "Dark", "count" : 17}, +{ "word" : "Delegate", "count" : 17}, +{ "word" : "Dropdown", "count" : 17}, +{ "word" : "Endings", "count" : 17}, +{ "word" : "Fix", "count" : 17}, +{ "word" : "Graphic", "count" : 17}, +{ "word" : "Header", "count" : 17}, +{ "word" : "Inset", "count" : 17}, +{ "word" : "Inside", "count" : 17}, +{ "word" : "Known", "count" : 17}, +{ "word" : "Last", "count" : 17}, +{ "word" : "Macros", "count" : 17}, +{ "word" : "Marked", "count" : 17}, +{ "word" : "Moving", "count" : 17}, +{ "word" : "NSCollection", "count" : 17}, +{ "word" : "NSWindow", "count" : 17}, +{ "word" : "Pattern", "count" : 17}, +{ "word" : "Rendition", "count" : 17}, +{ "word" : "Same", "count" : 17}, +{ "word" : "Spell", "count" : 17}, +{ "word" : "UIDrop", "count" : 17}, +{ "word" : "Vertical", "count" : 17}, +{ "word" : "aspect", "count" : 17}, +{ "word" : "bark", "count" : 17}, +{ "word" : "configure", "count" : 17}, +{ "word" : "containing", "count" : 17}, +{ "word" : "dict", "count" : 17}, +{ "word" : "dictionary", "count" : 17}, +{ "word" : "editing", "count" : 17}, +{ "word" : "free", "count" : 17}, +{ "word" : "mode", "count" : 17}, +{ "word" : "module", "count" : 17}, +{ "word" : "operation", "count" : 17}, +{ "word" : "translation", "count" : 17}, +{ "word" : "Bound", "count" : 18}, +{ "word" : "Candidate", "count" : 18}, +{ "word" : "Centered", "count" : 18}, +{ "word" : "Crash", "count" : 18}, +{ "word" : "Delimiter", "count" : 18}, +{ "word" : "MARK", "count" : 18}, +{ "word" : "NSError", "count" : 18}, +{ "word" : "Named", "count" : 18}, +{ "word" : "Param", "count" : 18}, +{ "word" : "Parameters", "count" : 18}, +{ "word" : "Pop", "count" : 18}, +{ "word" : "Prefix", "count" : 18}, +{ "word" : "Re", "count" : 18}, +{ "word" : "Scale", "count" : 18}, +{ "word" : "This", "count" : 18}, +{ "word" : "UITouch", "count" : 18}, +{ "word" : "be", "count" : 18}, +{ "word" : "convert", "count" : 18}, +{ "word" : "ctx", "count" : 18}, +{ "word" : "extend", "count" : 18}, +{ "word" : "f", "count" : 18}, +{ "word" : "float", "count" : 18}, +{ "word" : "guard", "count" : 18}, +{ "word" : "keyword", "count" : 18}, +{ "word" : "kinds", "count" : 18}, +{ "word" : "not", "count" : 18}, +{ "word" : "queue", "count" : 18}, +{ "word" : "schedule", "count" : 18}, +{ "word" : "section", "count" : 18}, +{ "word" : "tear", "count" : 18}, +{ "word" : "visible", "count" : 18}, +{ "word" : "CGContext", "count" : 19}, +{ "word" : "CName", "count" : 19}, +{ "word" : "Categorized", "count" : 19}, +{ "word" : "Cell", "count" : 19}, +{ "word" : "Checking", "count" : 19}, +{ "word" : "Deleting", "count" : 19}, +{ "word" : "Foo", "count" : 19}, +{ "word" : "Highlights", "count" : 19}, +{ "word" : "It", "count" : 19}, +{ "word" : "List", "count" : 19}, +{ "word" : "Modifier", "count" : 19}, +{ "word" : "Movement", "count" : 19}, +{ "word" : "Output", "count" : 19}, +{ "word" : "Queue", "count" : 19}, +{ "word" : "Reference", "count" : 19}, +{ "word" : "Return", "count" : 19}, +{ "word" : "Show", "count" : 19}, +{ "word" : "Sub", "count" : 19}, +{ "word" : "Wrap", "count" : 19}, +{ "word" : "an", "count" : 19}, +{ "word" : "arguments", "count" : 19}, +{ "word" : "candidate", "count" : 19}, +{ "word" : "clear", "count" : 19}, +{ "word" : "copy", "count" : 19}, +{ "word" : "disposition", "count" : 19}, +{ "word" : "drag", "count" : 19}, +{ "word" : "elements", "count" : 19}, +{ "word" : "enable", "count" : 19}, +{ "word" : "enum", "count" : 19}, +{ "word" : "expression", "count" : 19}, +{ "word" : "groups", "count" : 19}, +{ "word" : "icon", "count" : 19}, +{ "word" : "journal", "count" : 19}, +{ "word" : "log", "count" : 19}, +{ "word" : "margin", "count" : 19}, +{ "word" : "read", "count" : 19}, +{ "word" : "replacement", "count" : 19}, +{ "word" : "system", "count" : 19}, +{ "word" : "Bottom", "count" : 20}, +{ "word" : "Disposition", "count" : 20}, +{ "word" : "Enum", "count" : 20}, +{ "word" : "Hasher", "count" : 20}, +{ "word" : "Help", "count" : 20}, +{ "word" : "Level", "count" : 20}, +{ "word" : "NSOutline", "count" : 20}, +{ "word" : "Nested", "count" : 20}, +{ "word" : "Part", "count" : 20}, +{ "word" : "Preference", "count" : 20}, +{ "word" : "Protocol", "count" : 20}, +{ "word" : "Remove", "count" : 20}, +{ "word" : "Returns", "count" : 20}, +{ "word" : "Section", "count" : 20}, +{ "word" : "Substring", "count" : 20}, +{ "word" : "Translation", "count" : 20}, +{ "word" : "Version", "count" : 20}, +{ "word" : "aux", "count" : 20}, +{ "word" : "best", "count" : 20}, +{ "word" : "callback", "count" : 20}, +{ "word" : "callers", "count" : 20}, +{ "word" : "child", "count" : 20}, +{ "word" : "contains", "count" : 20}, +{ "word" : "g", "count" : 20}, +{ "word" : "hasher", "count" : 20}, +{ "word" : "height", "count" : 20}, +{ "word" : "initializer", "count" : 20}, +{ "word" : "leading", "count" : 20}, +{ "word" : "placeholder", "count" : 20}, +{ "word" : "that", "count" : 20}, +{ "word" : "Are", "count" : 21}, +{ "word" : "Availability", "count" : 21}, +{ "word" : "Blank", "count" : 21}, +{ "word" : "Brace", "count" : 21}, +{ "word" : "Changes", "count" : 21}, +{ "word" : "Defaults", "count" : 21}, +{ "word" : "Editable", "count" : 21}, +{ "word" : "Has", "count" : 21}, +{ "word" : "Incremental", "count" : 21}, +{ "word" : "Mark", "count" : 21}, +{ "word" : "NSPasteboard", "count" : 21}, +{ "word" : "One", "count" : 21}, +{ "word" : "Pad", "count" : 21}, +{ "word" : "Page", "count" : 21}, +{ "word" : "Region", "count" : 21}, +{ "word" : "Secondary", "count" : 21}, +{ "word" : "Suggestions", "count" : 21}, +{ "word" : "Switch", "count" : 21}, +{ "word" : "Trailing", "count" : 21}, +{ "word" : "Transient", "count" : 21}, +{ "word" : "UIDrag", "count" : 21}, +{ "word" : "UITable", "count" : 21}, +{ "word" : "animate", "count" : 21}, +{ "word" : "control", "count" : 21}, +{ "word" : "declaration", "count" : 21}, +{ "word" : "expand", "count" : 21}, +{ "word" : "global", "count" : 21}, +{ "word" : "has", "count" : 21}, +{ "word" : "hash", "count" : 21}, +{ "word" : "jump", "count" : 21}, +{ "word" : "numeric", "count" : 21}, +{ "word" : "project", "count" : 21}, +{ "word" : "scene", "count" : 21}, +{ "word" : "uninstall", "count" : 21}, +{ "word" : "Alignment", "count" : 22}, +{ "word" : "Debug", "count" : 22}, +{ "word" : "Feature", "count" : 22}, +{ "word" : "Fold", "count" : 22}, +{ "word" : "History", "count" : 22}, +{ "word" : "Leading", "count" : 22}, +{ "word" : "Match", "count" : 22}, +{ "word" : "Member", "count" : 22}, +{ "word" : "Project", "count" : 22}, +{ "word" : "Reason", "count" : 22}, +{ "word" : "Role", "count" : 22}, +{ "word" : "Search", "count" : 22}, +{ "word" : "Valid", "count" : 22}, +{ "word" : "autoclosure", "count" : 22}, +{ "word" : "call", "count" : 22}, +{ "word" : "cancel", "count" : 22}, +{ "word" : "count", "count" : 22}, +{ "word" : "e", "count" : 22}, +{ "word" : "hide", "count" : 22}, +{ "word" : "navigator", "count" : 22}, +{ "word" : "provider", "count" : 22}, +{ "word" : "reason", "count" : 22}, +{ "word" : "transform", "count" : 22}, +{ "word" : "Argument", "count" : 23}, +{ "word" : "Bytes", "count" : 23}, +{ "word" : "Char", "count" : 23}, +{ "word" : "IDEWorkspace", "count" : 23}, +{ "word" : "Layers", "count" : 23}, +{ "word" : "Open", "count" : 23}, +{ "word" : "Setting", "count" : 23}, +{ "word" : "Title", "count" : 23}, +{ "word" : "Views", "count" : 23}, +{ "word" : "Visualization", "count" : 23}, +{ "word" : "all", "count" : 23}, +{ "word" : "attributed", "count" : 23}, +{ "word" : "baz", "count" : 23}, +{ "word" : "bottom", "count" : 23}, +{ "word" : "closure", "count" : 23}, +{ "word" : "else", "count" : 23}, +{ "word" : "frame", "count" : 23}, +{ "word" : "pre", "count" : 23}, +{ "word" : "references", "count" : 23}, +{ "word" : "setup", "count" : 23}, +{ "word" : "shift", "count" : 23}, +{ "word" : "width", "count" : 23}, +{ "word" : "Custom", "count" : 24}, +{ "word" : "Definition", "count" : 24}, +{ "word" : "Delimiters", "count" : 24}, +{ "word" : "Description", "count" : 24}, +{ "word" : "Files", "count" : 24}, +{ "word" : "Generation", "count" : 24}, +{ "word" : "Internal", "count" : 24}, +{ "word" : "Note", "count" : 24}, +{ "word" : "Points", "count" : 24}, +{ "word" : "Style", "count" : 24}, +{ "word" : "Tab", "count" : 24}, +{ "word" : "Tests", "count" : 24}, +{ "word" : "Time", "count" : 24}, +{ "word" : "Timestamp", "count" : 24}, +{ "word" : "apply", "count" : 24}, +{ "word" : "handler", "count" : 24}, +{ "word" : "install", "count" : 24}, +{ "word" : "suggested", "count" : 24}, +{ "word" : "theme", "count" : 24}, +{ "word" : "verify", "count" : 24}, +{ "word" : "Background", "count" : 25}, +{ "word" : "Coverage", "count" : 25}, +{ "word" : "Doesnt", "count" : 25}, +{ "word" : "Framework", "count" : 25}, +{ "word" : "Matches", "count" : 25}, +{ "word" : "Matching", "count" : 25}, +{ "word" : "Preprocessor", "count" : 25}, +{ "word" : "by", "count" : 25}, +{ "word" : "category", "count" : 25}, +{ "word" : "column", "count" : 25}, +{ "word" : "instance", "count" : 25}, +{ "word" : "operator", "count" : 25}, +{ "word" : "property", "count" : 25}, +{ "word" : "relayout", "count" : 25}, +{ "word" : "Area", "count" : 26}, +{ "word" : "Begin", "count" : 26}, +{ "word" : "Callback", "count" : 26}, +{ "word" : "Characters", "count" : 26}, +{ "word" : "Hashable", "count" : 26}, +{ "word" : "Label", "count" : 26}, +{ "word" : "NSTable", "count" : 26}, +{ "word" : "Needs", "count" : 26}, +{ "word" : "Rects", "count" : 26}, +{ "word" : "Spacing", "count" : 26}, +{ "word" : "Static", "count" : 26}, +{ "word" : "Typing", "count" : 26}, +{ "word" : "U", "count" : 26}, +{ "word" : "Updated", "count" : 26}, +{ "word" : "When", "count" : 26}, +{ "word" : "character", "count" : 26}, +{ "word" : "col", "count" : 26}, +{ "word" : "defining", "count" : 26}, +{ "word" : "direction", "count" : 26}, +{ "word" : "interaction", "count" : 26}, +{ "word" : "markup", "count" : 26}, +{ "word" : "post", "count" : 26}, +{ "word" : "row", "count" : 26}, +{ "word" : "target", "count" : 26}, +{ "word" : "we", "count" : 26}, +{ "word" : "where", "count" : 26}, +{ "word" : "word", "count" : 26}, +{ "word" : "As", "count" : 27}, +{ "word" : "Assistant", "count" : 27}, +{ "word" : "Height", "count" : 27}, +{ "word" : "Landmarks", "count" : 27}, +{ "word" : "NSAccessibility", "count" : 27}, +{ "word" : "Names", "count" : 27}, +{ "word" : "Wrapping", "count" : 27}, +{ "word" : "enumerate", "count" : 27}, +{ "word" : "group", "count" : 27}, +{ "word" : "init", "count" : 27}, +{ "word" : "number", "count" : 27}, +{ "word" : "query", "count" : 27}, +{ "word" : "structured", "count" : 27}, +{ "word" : "translator", "count" : 27}, +{ "word" : "After", "count" : 28}, +{ "word" : "Equivalent", "count" : 28}, +{ "word" : "Finish", "count" : 28}, +{ "word" : "Formatter", "count" : 28}, +{ "word" : "Global", "count" : 28}, +{ "word" : "Insets", "count" : 28}, +{ "word" : "Pass", "count" : 28}, +{ "word" : "Replacement", "count" : 28}, +{ "word" : "Translator", "count" : 28}, +{ "word" : "block", "count" : 28}, +{ "word" : "dragging", "count" : 28}, +{ "word" : "matching", "count" : 28}, +{ "word" : "meow", "count" : 28}, +{ "word" : "message", "count" : 28}, +{ "word" : "response", "count" : 28}, +{ "word" : "send", "count" : 28}, +{ "word" : "touches", "count" : 28}, +{ "word" : "workspace", "count" : 28}, +{ "word" : "A", "count" : 29}, +{ "word" : "CAAction", "count" : 29}, +{ "word" : "Commands", "count" : 29}, +{ "word" : "Drop", "count" : 29}, +{ "word" : "Import", "count" : 29}, +{ "word" : "Literal", "count" : 29}, +{ "word" : "Space", "count" : 29}, +{ "word" : "Throws", "count" : 29}, +{ "word" : "command", "count" : 29}, +{ "word" : "lines", "count" : 29}, +{ "word" : "mouse", "count" : 29}, +{ "word" : "nreturn", "count" : 29}, +{ "word" : "options", "count" : 29}, +{ "word" : "selected", "count" : 29}, +{ "word" : "this", "count" : 29}, +{ "word" : "Auxiliary", "count" : 30}, +{ "word" : "Declarator", "count" : 30}, +{ "word" : "Effect", "count" : 30}, +{ "word" : "Empty", "count" : 30}, +{ "word" : "Encoding", "count" : 30}, +{ "word" : "Icon", "count" : 30}, +{ "word" : "Macro", "count" : 30}, +{ "word" : "NSString", "count" : 30}, +{ "word" : "Responder", "count" : 30}, +{ "word" : "Unkeyed", "count" : 30}, +{ "word" : "adjust", "count" : 30}, +{ "word" : "completions", "count" : 30}, +{ "word" : "formatting", "count" : 30}, +{ "word" : "inserted", "count" : 30}, +{ "word" : "into", "count" : 30}, +{ "word" : "search", "count" : 30}, +{ "word" : "utility", "count" : 30}, +{ "word" : "x", "count" : 30}, +{ "word" : "Close", "count" : 31}, +{ "word" : "Groups", "count" : 31}, +{ "word" : "Interval", "count" : 31}, +{ "word" : "Multi", "count" : 31}, +{ "word" : "Navigator", "count" : 31}, +{ "word" : "Property", "count" : 31}, +{ "word" : "Split", "count" : 31}, +{ "word" : "Wrapped", "count" : 31}, +{ "word" : "char", "count" : 31}, +{ "word" : "delete", "count" : 31}, +{ "word" : "my", "count" : 31}, +{ "word" : "prefix", "count" : 31}, +{ "word" : "quick", "count" : 31}, +{ "word" : "s", "count" : 31}, +{ "word" : "user", "count" : 31}, +{ "word" : "Edge", "count" : 32}, +{ "word" : "Modify", "count" : 32}, +{ "word" : "Statement", "count" : 32}, +{ "word" : "UIGesture", "count" : 32}, +{ "word" : "close", "count" : 32}, +{ "word" : "draw", "count" : 32}, +{ "word" : "formatter", "count" : 32}, +{ "word" : "import", "count" : 32}, +{ "word" : "uid", "count" : 32}, +{ "word" : "utf", "count" : 32}, +{ "word" : "C", "count" : 33}, +{ "word" : "CMBlock", "count" : 33}, +{ "word" : "Comments", "count" : 33}, +{ "word" : "Filter", "count" : 33}, +{ "word" : "Journal", "count" : 33}, +{ "word" : "Overrides", "count" : 33}, +{ "word" : "Parameter", "count" : 33}, +{ "word" : "Scalar", "count" : 33}, +{ "word" : "Substitution", "count" : 33}, +{ "word" : "application", "count" : 33}, +{ "word" : "base", "count" : 33}, +{ "word" : "current", "count" : 33}, +{ "word" : "documentation", "count" : 33}, +{ "word" : "first", "count" : 33}, +{ "word" : "functionality", "count" : 33}, +{ "word" : "highlight", "count" : 33}, +{ "word" : "id", "count" : 33}, +{ "word" : "results", "count" : 33}, +{ "word" : "some", "count" : 33}, +{ "word" : "Continuation", "count" : 34}, +{ "word" : "Diagnostics", "count" : 34}, +{ "word" : "Non", "count" : 34}, +{ "word" : "Refactoring", "count" : 34}, +{ "word" : "Viewport", "count" : 34}, +{ "word" : "adjusted", "count" : 34}, +{ "word" : "append", "count" : 34}, +{ "word" : "begin", "count" : 34}, +{ "word" : "node", "count" : 34}, +{ "word" : "observer", "count" : 34}, +{ "word" : "old", "count" : 34}, +{ "word" : "Category", "count" : 35}, +{ "word" : "Dragging", "count" : 35}, +{ "word" : "NSDocument", "count" : 35}, +{ "word" : "it", "count" : 35}, +{ "word" : "observed", "count" : 35}, +{ "word" : "or", "count" : 35}, +{ "word" : "stop", "count" : 35}, +{ "word" : "syntaxcolor", "count" : 35}, +{ "word" : "Copy", "count" : 36}, +{ "word" : "Formatting", "count" : 36}, +{ "word" : "IDEIndex", "count" : 36}, +{ "word" : "Lozenge", "count" : 36}, +{ "word" : "Placement", "count" : 36}, +{ "word" : "R", "count" : 36}, +{ "word" : "Row", "count" : 36}, +{ "word" : "Scoring", "count" : 36}, +{ "word" : "Settings", "count" : 36}, +{ "word" : "Syntax", "count" : 36}, +{ "word" : "Unicode", "count" : 36}, +{ "word" : "accessor", "count" : 36}, +{ "word" : "gutter", "count" : 36}, +{ "word" : "named", "count" : 36}, +{ "word" : "object", "count" : 36}, +{ "word" : "quickly", "count" : 36}, +{ "word" : "replace", "count" : 36}, +{ "word" : "Arg", "count" : 37}, +{ "word" : "Configuration", "count" : 37}, +{ "word" : "Map", "count" : 37}, +{ "word" : "Relative", "count" : 37}, +{ "word" : "Tokens", "count" : 37}, +{ "word" : "characters", "count" : 37}, +{ "word" : "container", "count" : 37}, +{ "word" : "label", "count" : 37}, +{ "word" : "path", "count" : 37}, +{ "word" : "ranges", "count" : 37}, +{ "word" : "Arrow", "count" : 38}, +{ "word" : "Batch", "count" : 38}, +{ "word" : "Children", "count" : 38}, +{ "word" : "IDESource", "count" : 38}, +{ "word" : "Self", "count" : 38}, +{ "word" : "Service", "count" : 38}, +{ "word" : "gesture", "count" : 38}, +{ "word" : "process", "count" : 38}, +{ "word" : "reset", "count" : 38}, +{ "word" : "toggle", "count" : 38}, +{ "word" : "var", "count" : 38}, +{ "word" : "Inserting", "count" : 39}, +{ "word" : "Keystroke", "count" : 39}, +{ "word" : "Markup", "count" : 39}, +{ "word" : "New", "count" : 39}, +{ "word" : "Transaction", "count" : 39}, +{ "word" : "buffer", "count" : 39}, +{ "word" : "double", "count" : 39}, +{ "word" : "encode", "count" : 39}, +{ "word" : "using", "count" : 39}, +{ "word" : "APINotes", "count" : 40}, +{ "word" : "Field", "count" : 40}, +{ "word" : "NSDragging", "count" : 40}, +{ "word" : "Right", "count" : 40}, +{ "word" : "map", "count" : 40}, +{ "word" : "scope", "count" : 40}, +{ "word" : "Mutable", "count" : 41}, +{ "word" : "Optional", "count" : 41}, +{ "word" : "Overlay", "count" : 41}, +{ "word" : "UTF", "count" : 41}, +{ "word" : "invalidate", "count" : 41}, +{ "word" : "session", "count" : 41}, +{ "word" : "Args", "count" : 42}, +{ "word" : "CType", "count" : 42}, +{ "word" : "Call", "count" : 42}, +{ "word" : "Dictionary", "count" : 42}, +{ "word" : "Load", "count" : 42}, +{ "word" : "Raw", "count" : 42}, +{ "word" : "Shift", "count" : 42}, +{ "word" : "result", "count" : 42}, +{ "word" : "size", "count" : 42}, +{ "word" : "Aux", "count" : 43}, +{ "word" : "Case", "count" : 43}, +{ "word" : "Gesture", "count" : 43}, +{ "word" : "Insertion", "count" : 43}, +{ "word" : "Or", "count" : 43}, +{ "word" : "and", "count" : 43}, +{ "word" : "i", "count" : 43}, +{ "word" : "modifier", "count" : 43}, +{ "word" : "offset", "count" : 43}, +{ "word" : "r", "count" : 43}, +{ "word" : "rect", "count" : 43}, +{ "word" : "Elements", "count" : 44}, +{ "word" : "Invalidate", "count" : 44}, +{ "word" : "NSView", "count" : 44}, +{ "word" : "Whitespace", "count" : 44}, +{ "word" : "filter", "count" : 44}, +{ "word" : "items", "count" : 44}, +{ "word" : "outline", "count" : 44}, +{ "word" : "types", "count" : 44}, +{ "word" : "Annotations", "count" : 45}, +{ "word" : "Float", "count" : 45}, +{ "word" : "NSMutable", "count" : 45}, +{ "word" : "Needed", "count" : 45}, +{ "word" : "change", "count" : 45}, +{ "word" : "create", "count" : 45}, +{ "word" : "present", "count" : 45}, +{ "word" : "Active", "count" : 46}, +{ "word" : "Click", "count" : 46}, +{ "word" : "Descriptor", "count" : 46}, +{ "word" : "Request", "count" : 46}, +{ "word" : "arg", "count" : 46}, +{ "word" : "c", "count" : 46}, +{ "word" : "hello", "count" : 46}, +{ "word" : "landmarks", "count" : 46}, +{ "word" : "level", "count" : 46}, +{ "word" : "validate", "count" : 46}, +{ "word" : "Assert", "count" : 47}, +{ "word" : "Parse", "count" : 47}, +{ "word" : "Replace", "count" : 47}, +{ "word" : "Snippet", "count" : 47}, +{ "word" : "Start", "count" : 47}, +{ "word" : "The", "count" : 47}, +{ "word" : "UIText", "count" : 47}, +{ "word" : "default", "count" : 47}, +{ "word" : "Direction", "count" : 48}, +{ "word" : "IDs", "count" : 48}, +{ "word" : "Keyword", "count" : 48}, +{ "word" : "animated", "count" : 48}, +{ "word" : "str", "count" : 48}, +{ "word" : "xcode", "count" : 48}, +{ "word" : "Add", "count" : 49}, +{ "word" : "Control", "count" : 49}, +{ "word" : "Forward", "count" : 49}, +{ "word" : "Handler", "count" : 49}, +{ "word" : "Offset", "count" : 49}, +{ "word" : "Unsafe", "count" : 49}, +{ "word" : "as", "count" : 49}, +{ "word" : "indent", "count" : 49}, +{ "word" : "raw", "count" : 49}, +{ "word" : "Backward", "count" : 50}, +{ "word" : "Contents", "count" : 50}, +{ "word" : "Mode", "count" : 50}, +{ "word" : "Undo", "count" : 50}, +{ "word" : "array", "count" : 50}, +{ "word" : "bar", "count" : 50}, +{ "word" : "Comment", "count" : 51}, +{ "word" : "No", "count" : 51}, +{ "word" : "next", "count" : 51}, +{ "word" : "Collection", "count" : 52}, +{ "word" : "Beginning", "count" : 53}, +{ "word" : "Default", "count" : 53}, +{ "word" : "Error", "count" : 53}, +{ "word" : "NSAttributed", "count" : 53}, +{ "word" : "NSRect", "count" : 53}, +{ "word" : "Rename", "count" : 53}, +{ "word" : "collection", "count" : 53}, +{ "word" : "register", "count" : 53}, +{ "word" : "Attributes", "count" : 54}, +{ "word" : "Container", "count" : 54}, +{ "word" : "Image", "count" : 54}, +{ "word" : "Margin", "count" : 54}, +{ "word" : "Process", "count" : 54}, +{ "word" : "annotation", "count" : 54}, +{ "word" : "discardable", "count" : 54}, +{ "word" : "parameter", "count" : 54}, +{ "word" : "request", "count" : 54}, +{ "word" : "Log", "count" : 55}, +{ "word" : "Scoped", "count" : 55}, +{ "word" : "on", "count" : 55}, +{ "word" : "variant", "count" : 55}, +{ "word" : "CGSize", "count" : 56}, +{ "word" : "Perform", "count" : 56}, +{ "word" : "body", "count" : 56}, +{ "word" : "end", "count" : 56}, +{ "word" : "ids", "count" : 56}, +{ "word" : "By", "count" : 57}, +{ "word" : "Previous", "count" : 57}, +{ "word" : "Visible", "count" : 57}, +{ "word" : "can", "count" : 57}, +{ "word" : "menu", "count" : 57}, +{ "word" : "other", "count" : 57}, +{ "word" : "state", "count" : 57}, +{ "word" : "First", "count" : 58}, +{ "word" : "Interface", "count" : 58}, +{ "word" : "Left", "count" : 58}, +{ "word" : "Multiple", "count" : 58}, +{ "word" : "NSMenu", "count" : 58}, +{ "word" : "User", "count" : 58}, +{ "word" : "include", "count" : 58}, +{ "word" : "window", "count" : 58}, +{ "word" : "Count", "count" : 59}, +{ "word" : "Width", "count" : 59}, +{ "word" : "Cursor", "count" : 60}, +{ "word" : "Paragraph", "count" : 60}, +{ "word" : "font", "count" : 60}, +{ "word" : "landmark", "count" : 60}, +{ "word" : "parse", "count" : 60}, +{ "word" : "ref", "count" : 60}, +{ "word" : "Array", "count" : 61}, +{ "word" : "Interaction", "count" : 61}, +{ "word" : "Selected", "count" : 61}, +{ "word" : "parent", "count" : 61}, +{ "word" : "value", "count" : 61}, +{ "word" : "Next", "count" : 62}, +{ "word" : "Pointer", "count" : 62}, +{ "word" : "reference", "count" : 62}, +{ "word" : "System", "count" : 63}, +{ "word" : "b", "count" : 63}, +{ "word" : "enclosing", "count" : 63}, +{ "word" : "make", "count" : 63}, +{ "word" : "NSKey", "count" : 64}, +{ "word" : "code", "count" : 64}, +{ "word" : "notification", "count" : 64}, +{ "word" : "Accessibility", "count" : 65}, +{ "word" : "Documentation", "count" : 65}, +{ "word" : "Expression", "count" : 65}, +{ "word" : "Flags", "count" : 65}, +{ "word" : "Is", "count" : 65}, +{ "word" : "scroll", "count" : 65}, +{ "word" : "syntax", "count" : 65}, +{ "word" : "If", "count" : 66}, +{ "word" : "Number", "count" : 66}, +{ "word" : "Structured", "count" : 66}, +{ "word" : "action", "count" : 66}, +{ "word" : "symbol", "count" : 66}, +{ "word" : "Current", "count" : 67}, +{ "word" : "kind", "count" : 68}, +{ "word" : "Class", "count" : 69}, +{ "word" : "DVTText", "count" : 69}, +{ "word" : "Status", "count" : 69}, +{ "word" : "Completions", "count" : 70}, +{ "word" : "Group", "count" : 70}, +{ "word" : "Selector", "count" : 70}, +{ "word" : "Something", "count" : 70}, +{ "word" : "insert", "count" : 70}, +{ "word" : "Double", "count" : 71}, +{ "word" : "Frame", "count" : 71}, +{ "word" : "Placeholder", "count" : 71}, +{ "word" : "find", "count" : 71}, +{ "word" : "new", "count" : 71}, +{ "word" : "table", "count" : 71}, +{ "word" : "Update", "count" : 72}, +{ "word" : "Accessory", "count" : 73}, +{ "word" : "Diagnostic", "count" : 73}, +{ "word" : "Objc", "count" : 73}, +{ "word" : "Session", "count" : 73}, +{ "word" : "Mouse", "count" : 74}, +{ "word" : "Types", "count" : 74}, +{ "word" : "color", "count" : 74}, +{ "word" : "structure", "count" : 74}, +{ "word" : "Performance", "count" : 75}, +{ "word" : "Panel", "count" : 76}, +{ "word" : "Recognizer", "count" : 76}, +{ "word" : "Results", "count" : 76}, +{ "word" : "mutating", "count" : 76}, +{ "word" : "Drag", "count" : 77}, +{ "word" : "Button", "count" : 78}, +{ "word" : "DVTSource", "count" : 78}, +{ "word" : "start", "count" : 78}, +{ "word" : "Observing", "count" : 80}, +{ "word" : "Column", "count" : 81}, +{ "word" : "Display", "count" : 81}, +{ "word" : "Keypath", "count" : 81}, +{ "word" : "Single", "count" : 81}, +{ "word" : "Consumer", "count" : 82}, +{ "word" : "display", "count" : 82}, +{ "word" : "do", "count" : 82}, +{ "word" : "functions", "count" : 82}, +{ "word" : "Pane", "count" : 83}, +{ "word" : "Quickly", "count" : 83}, +{ "word" : "Scroll", "count" : 83}, +{ "word" : "Down", "count" : 84}, +{ "word" : "Size", "count" : 84}, +{ "word" : "element", "count" : 84}, +{ "word" : "Scope", "count" : 85}, +{ "word" : "Lines", "count" : 86}, +{ "word" : "Module", "count" : 86}, +{ "word" : "Query", "count" : 86}, +{ "word" : "accessibility", "count" : 86}, +{ "word" : "Indent", "count" : 87}, +{ "word" : "Will", "count" : 87}, +{ "word" : "show", "count" : 87}, +{ "word" : "Highlight", "count" : 88}, +{ "word" : "Decl", "count" : 89}, +{ "word" : "Character", "count" : 90}, +{ "word" : "Extension", "count" : 90}, +{ "word" : "On", "count" : 90}, +{ "word" : "Override", "count" : 91}, +{ "word" : "Rect", "count" : 91}, +{ "word" : "Theme", "count" : 91}, +{ "word" : "lhs", "count" : 91}, +{ "word" : "Cpp", "count" : 92}, +{ "word" : "Paste", "count" : 92}, +{ "word" : "move", "count" : 92}, +{ "word" : "Word", "count" : 93}, +{ "word" : "Items", "count" : 94}, +{ "word" : "rhs", "count" : 94}, +{ "word" : "language", "count" : 95}, +{ "word" : "location", "count" : 96}, +{ "word" : "Attribute", "count" : 98}, +{ "word" : "UID", "count" : 98}, +{ "word" : "Composite", "count" : 99}, +{ "word" : "Info", "count" : 99}, +{ "word" : "Observer", "count" : 99}, +{ "word" : "Test", "count" : 99}, +{ "word" : "Target", "count" : 100}, +{ "word" : "point", "count" : 100}, +{ "word" : "NSEvent", "count" : 101}, +{ "word" : "Point", "count" : 101}, +{ "word" : "url", "count" : 101}, +{ "word" : "Buffer", "count" : 102}, +{ "word" : "UInt", "count" : 102}, +{ "word" : "Notification", "count" : 103}, +{ "word" : "Quick", "count" : 103}, +{ "word" : "editor", "count" : 103}, +{ "word" : "get", "count" : 104}, +{ "word" : "End", "count" : 105}, +{ "word" : "case", "count" : 106}, +{ "word" : "navigation", "count" : 106}, +{ "word" : "Operation", "count" : 107}, +{ "word" : "Editing", "count" : 108}, +{ "word" : "Insert", "count" : 110}, +{ "word" : "document", "count" : 110}, +{ "word" : "select", "count" : 111}, +{ "word" : "Move", "count" : 112}, +{ "word" : "selection", "count" : 112}, +{ "word" : "will", "count" : 113}, +{ "word" : "FALSE", "count" : 115}, +{ "word" : "index", "count" : 115}, +{ "word" : "Suggestion", "count" : 116}, +{ "word" : "Vi", "count" : 116}, +{ "word" : "XCTAssert", "count" : 117}, +{ "word" : "of", "count" : 117}, +{ "word" : "IBAction", "count" : 119}, +{ "word" : "identifier", "count" : 119}, +{ "word" : "token", "count" : 119}, +{ "word" : "int", "count" : 120}, +{ "word" : "All", "count" : 121}, +{ "word" : "Declaration", "count" : 121}, +{ "word" : "sourcekit", "count" : 121}, +{ "word" : "should", "count" : 122}, +{ "word" : "Obj", "count" : 123}, +{ "word" : "Select", "count" : 123}, +{ "word" : "Window", "count" : 123}, +{ "word" : "Equal", "count" : 124}, +{ "word" : "remove", "count" : 124}, +{ "word" : "t", "count" : 128}, +{ "word" : "decl", "count" : 129}, +{ "word" : "optional", "count" : 130}, +{ "word" : "handle", "count" : 132}, +{ "word" : "T", "count" : 133}, +{ "word" : "Bindings", "count" : 134}, +{ "word" : "CGRect", "count" : 134}, +{ "word" : "Gutter", "count" : 134}, +{ "word" : "internal", "count" : 134}, +{ "word" : "Kind", "count" : 135}, +{ "word" : "Structure", "count" : 136}, +{ "word" : "observe", "count" : 136}, +{ "word" : "Provider", "count" : 138}, +{ "word" : "Ranges", "count" : 140}, +{ "word" : "Method", "count" : 141}, +{ "word" : "Up", "count" : 141}, +{ "word" : "inout", "count" : 141}, +{ "word" : "Bar", "count" : 142}, +{ "word" : "Function", "count" : 143}, +{ "word" : "from", "count" : 147}, +{ "word" : "Delete", "count" : 148}, +{ "word" : "From", "count" : 149}, +{ "word" : "method", "count" : 149}, +{ "word" : "Location", "count" : 150}, +{ "word" : "if", "count" : 152}, +{ "word" : "Object", "count" : 154}, +{ "word" : "Minimap", "count" : 156}, +{ "word" : "Change", "count" : 157}, +{ "word" : "Block", "count" : 160}, +{ "word" : "ID", "count" : 160}, +{ "word" : "the", "count" : 170}, +{ "word" : "Changed", "count" : 172}, +{ "word" : "Options", "count" : 174}, +{ "word" : "add", "count" : 174}, +{ "word" : "Action", "count" : 176}, +{ "word" : "Menu", "count" : 176}, +{ "word" : "Layer", "count" : 177}, +{ "word" : "Value", "count" : 179}, +{ "word" : "Response", "count" : 180}, +{ "word" : "return", "count" : 181}, +{ "word" : "file", "count" : 182}, +{ "word" : "let", "count" : 182}, +{ "word" : "text", "count" : 182}, +{ "word" : "CGPoint", "count" : 185}, +{ "word" : "NSRange", "count" : 185}, +{ "word" : "completion", "count" : 186}, +{ "word" : "content", "count" : 186}, +{ "word" : "Landmark", "count" : 190}, +{ "word" : "escaping", "count" : 192}, +{ "word" : "TRUE", "count" : 194}, +{ "word" : "Set", "count" : 196}, +{ "word" : "State", "count" : 198}, +{ "word" : "Void", "count" : 198}, +{ "word" : "Node", "count" : 199}, +{ "word" : "Edit", "count" : 201}, +{ "word" : "Path", "count" : 205}, +{ "word" : "class", "count" : 210}, +{ "word" : "Func", "count" : 212}, +{ "word" : "key", "count" : 213}, +{ "word" : "Def", "count" : 216}, +{ "word" : "event", "count" : 221}, +{ "word" : "Color", "count" : 223}, +{ "word" : "set", "count" : 224}, +{ "word" : "Kit", "count" : 225}, +{ "word" : "view", "count" : 225}, +{ "word" : "Language", "count" : 227}, +{ "word" : "lang", "count" : 229}, +{ "word" : "perform", "count" : 233}, +{ "word" : "And", "count" : 236}, +{ "word" : "Navigation", "count" : 238}, +{ "word" : "swift", "count" : 239}, +{ "word" : "type", "count" : 241}, +{ "word" : "Font", "count" : 242}, +{ "word" : "Find", "count" : 247}, +{ "word" : "Of", "count" : 252}, +{ "word" : "to", "count" : 252}, +{ "word" : "is", "count" : 257}, +{ "word" : "At", "count" : 259}, +{ "word" : "File", "count" : 259}, +{ "word" : "nil", "count" : 266}, +{ "word" : "Controller", "count" : 267}, +{ "word" : "CGFloat", "count" : 268}, +{ "word" : "fileprivate", "count" : 268}, +{ "word" : "Event", "count" : 272}, +{ "word" : "Did", "count" : 275}, +{ "word" : "Index", "count" : 275}, +{ "word" : "Token", "count" : 276}, +{ "word" : "Annotation", "count" : 280}, +{ "word" : "Document", "count" : 286}, +{ "word" : "Result", "count" : 299}, +{ "word" : "Completion", "count" : 301}, +{ "word" : "did", "count" : 306}, +{ "word" : "item", "count" : 313}, +{ "word" : "layout", "count" : 314}, +{ "word" : "name", "count" : 317}, +{ "word" : "update", "count" : 334}, +{ "word" : "Element", "count" : 340}, +{ "word" : "context", "count" : 344}, +{ "word" : "at", "count" : 346}, +{ "word" : "With", "count" : 365}, +{ "word" : "Content", "count" : 373}, +{ "word" : "a", "count" : 373}, +{ "word" : "URL", "count" : 380}, +{ "word" : "Key", "count" : 383}, +{ "word" : "Name", "count" : 387}, +{ "word" : "string", "count" : 391}, +{ "word" : "Swift", "count" : 400}, +{ "word" : "In", "count" : 417}, +{ "word" : "SMSource", "count" : 429}, +{ "word" : "Manager", "count" : 435}, +{ "word" : "Identifier", "count" : 437}, +{ "word" : "with", "count" : 438}, +{ "word" : "objc", "count" : 440}, +{ "word" : "position", "count" : 440}, +{ "word" : "Code", "count" : 466}, +{ "word" : "Context", "count" : 480}, +{ "word" : "Layout", "count" : 483}, +{ "word" : "foo", "count" : 483}, +{ "word" : "Data", "count" : 495}, +{ "word" : "Command", "count" : 508}, +{ "word" : "To", "count" : 518}, +{ "word" : "Selection", "count" : 534}, +{ "word" : "in", "count" : 542}, +{ "word" : "Text", "count" : 546}, +{ "word" : "open", "count" : 564}, +{ "word" : "Model", "count" : 576}, +{ "word" : "data", "count" : 581}, +{ "word" : "Cache", "count" : 590}, +{ "word" : "range", "count" : 602}, +{ "word" : "sender", "count" : 632}, +{ "word" : "source", "count" : 635}, +{ "word" : "Symbol", "count" : 736}, +{ "word" : "line", "count" : 773}, +{ "word" : "Item", "count" : 812}, +{ "word" : "function", "count" : 824}, +{ "word" : "Position", "count" : 846}, +{ "word" : "for", "count" : 879}, +{ "word" : "Any", "count" : 905}, +{ "word" : "For", "count" : 928}, +{ "word" : "n", "count" : 953}, +{ "word" : "Type", "count" : 1063}, +{ "word" : "Line", "count" : 1122}, +{ "word" : "Int", "count" : 1282}, +{ "word" : "Bool", "count" : 1459}, +{ "word" : "View", "count" : 1555}, +{ "word" : "String", "count" : 1753}, +{ "word" : "test", "count" : 1917}, +{ "word" : "Range", "count" : 2311}, +{ "word" : "Editor", "count" : 3726}, +{ "word" : "Source", "count" : 4647}, +] diff --git a/Sources/CompletionScoringTestSupport/RandomSeed.plist b/Sources/CompletionScoringTestSupport/RandomSeed.plist new file mode 100644 index 000000000..4bb648ec5 --- /dev/null +++ b/Sources/CompletionScoringTestSupport/RandomSeed.plist @@ -0,0 +1,1030 @@ + + + + + 10649861844062857673 + 9060052091246972257 + 12942090708244661392 + 9933494248155016441 + 10828989690894343768 + 12788493555305265074 + 488913797917615037 + 17822819483841388785 + 16041325000510672802 + 17190357727871864693 + 2604045150856744782 + 3838047026243215059 + 16401602770669964302 + 17107743994455227755 + 15434291936037106882 + 4225037278253001179 + 2226084105488369337 + 4687545976950943822 + 18300466681609490739 + 1980023098261872498 + 17670050748207816560 + 12736629629229369041 + 17534768455463174174 + 8898732241997181651 + 11931833952676904910 + 4180390725280484058 + 17205636073021493376 + 8276422979551858588 + 11016946354289893893 + 12308495956251452848 + 7263476329554197015 + 16648149319419784874 + 6910896050161240625 + 4537394396171061871 + 14330771505108052901 + 2648492956127761911 + 4573649173901336171 + 14309751716159813221 + 5292378636516124469 + 4275765076679543001 + 16800899818181450055 + 18046600854094793356 + 3328010366260167308 + 18321147642883125694 + 6369084976403485091 + 588331132772749814 + 11797866875121243596 + 11370890767156732586 + 7719936951740554188 + 3556339505485058626 + 17945163430954770531 + 16384587621745106164 + 14177709071124149204 + 16389318410635712785 + 12629884384090305288 + 3952432847401018371 + 8887921331038563277 + 9224845742834590146 + 7047212806667268568 + 5387596695274976479 + 1131203715221761975 + 2995057563514828948 + 558281077740462941 + 5367201933815075486 + 18027555175583703203 + 3815931685421562694 + 15880990512495071458 + 11896350560114355651 + 15814670821480969269 + 10935248045238249312 + 14181477726458506552 + 13315947572172874533 + 2470569774214883886 + 2798813742852898826 + 17614405024579320401 + 6523750826029322289 + 6756800893765504828 + 9818274192037245886 + 5117070528299740587 + 13814322025010542269 + 10356863184679782778 + 14975879377646477909 + 2350268543890682019 + 6631595411582422424 + 13277607298226234479 + 10385880706748317705 + 3660035258487782819 + 6008288383785847474 + 246601306717851796 + 848618438465728844 + 2070517310903822636 + 7432124920080869709 + 687381512677760101 + 12015022105020395012 + 3475230368836172952 + 16455552951368856545 + 15441048807568448572 + 8546565033941286823 + 16604558369994954083 + 11511725767193317464 + 10700027132809754525 + 10125005576366217941 + 14173153666902620086 + 18435111687658672088 + 6081171684353748341 + 7117220862167305877 + 8715329109917373981 + 14576066639047453787 + 6819838309247178119 + 5315591605791168058 + 4183658152695964138 + 15850723641679793030 + 3260394523996062592 + 14708104201405533466 + 16750871475452877735 + 1501301636767725184 + 4175247676013603295 + 9365772035441220932 + 2493974198448035110 + 16370832881004661625 + 7232607699507705655 + 7089957862226922504 + 8889564998214887059 + 13250930397927823913 + 7603648579649087770 + 8806788810289802956 + 10485507589128383963 + 8796912621312452297 + 7944218858783999674 + 13592161546327668189 + 15835726225464933071 + 15756432662032529541 + 12073430065772685012 + 6952632564689999453 + 11249630342032732808 + 2509867731429781047 + 16904639689230170389 + 17612059305728605918 + 13851406172513444310 + 5483410043989408105 + 6938766405332021603 + 1510949420482395497 + 2996629394982073843 + 10288087186209705197 + 15778600019175082509 + 1497188676751912940 + 12456166283648243832 + 4328215192563713093 + 16063362166699614075 + 224440550801654658 + 467173145809158449 + 6051582188300983325 + 1521644966635562847 + 5349583029163228262 + 8803258351493233214 + 6207477265726665160 + 12490477206949640997 + 12173050753365171247 + 16627447209970552133 + 4899007879465179257 + 1032877983252151766 + 14800195005248586699 + 103714118666334821 + 6331463130237455963 + 8023292035116829472 + 12743427328631916460 + 10893743866060136956 + 11784908726336611686 + 9272180451441530572 + 5226640388588021031 + 5282421459110945544 + 4603209530857826057 + 3405454570302181247 + 2652317086563278175 + 16267939348686284725 + 18037428696829987574 + 15592816560914422821 + 8812378943602908405 + 14025012160250685737 + 15162339649807406865 + 15215894536809080237 + 10167694656226448375 + 14702025245317465605 + 9372459083077415041 + 818358333489715854 + 14117643407301722750 + 8879804017142196824 + 112533076176097978 + 1511090644119764320 + 8225431350377428647 + 6199691980930380308 + 6828841551981237016 + 17750823230854363753 + 2467076291276307182 + 12215749135069455309 + 5638571524478586382 + 15678666311027847249 + 4143399646329005882 + 16113772543508599165 + 8209040153748446089 + 15112564987413698859 + 6172570544118441475 + 10322741172264022672 + 803053488013042119 + 4595786559820044176 + 15471468821072007798 + 11236864219566698735 + 17624017015062092641 + 14399242371297004032 + 11896041473242596087 + 11618110437649791830 + 14247925393773410211 + 17756227468854180793 + 16771115508141070992 + 2234447464881468255 + 9617656292538057974 + 5370920711691898768 + 2424105781649600271 + 1492710966453179177 + 136463354103358456 + 770480694893157328 + 14250317640649090542 + 8448646640351200030 + 1161853657331519536 + 4401159506571376371 + 6591894851150470146 + 5922434182959282354 + 6217920520259148784 + 15347344960336749413 + 13504388544471169084 + 13541491650812014632 + 1192070502729533005 + 16359984411140186755 + 12842056301276820267 + 5290884053919160288 + 1955099608780971257 + 3106135892764740981 + 1085016533782390853 + 17608170015828607165 + 13156774632397276808 + 14073268027679629308 + 3715854401229574121 + 13116947759532507420 + 2876934206571122475 + 14076682054552377635 + 12243305646559136372 + 12223706178285816638 + 14979559968528001213 + 12335417939297159304 + 2936252503089252940 + 3747382846822585987 + 15095295351041827739 + 5591514828918870630 + 7683111237086780725 + 17580409963527887307 + 3464617359382551874 + 10756855216724740053 + 15632332098803710275 + 17710282918878009389 + 10723606802310966690 + 7757433379777905208 + 8493023912912638493 + 16878616212254342421 + 16309339687315968312 + 5312406456728738908 + 6326506451277319174 + 8667727879590816023 + 1494069411588951712 + 17297971643510650192 + 16917457951535409155 + 10664288556384772179 + 14708376270460122483 + 6181317130280679048 + 870273264114765336 + 5525322326233390453 + 17397676358668710084 + 15577340719173711292 + 14960804116168411205 + 4187771741316334282 + 6445682767178346728 + 7330689919879062999 + 2190542648006954640 + 751295493049147875 + 14946585476217475215 + 4359629194490935816 + 1717022239925159845 + 8096410117681086795 + 13478634282918720960 + 16681040550139062010 + 4278750200559248307 + 15200667783814360428 + 5476254718955324267 + 13455357500767250649 + 10131480236589848261 + 12311081712143503650 + 2451208355928096722 + 3404070383063701936 + 1922936335652107237 + 8237728127754705548 + 12098220285188531801 + 5937916602119842503 + 16175212903699578327 + 12021790035614654024 + 12640112880082548543 + 9988825613639928377 + 222609074448911387 + 1463521475844014988 + 4814437674650683769 + 16663544195197391006 + 6776010347659523905 + 10762042953762752045 + 5466415224686378976 + 14096361420610155366 + 15984576656958250391 + 1400516108101495466 + 5363503825487985388 + 1150631902970999264 + 10767412659093418454 + 18322557570588838336 + 16881614450553938560 + 3338835856882095887 + 11006002028040042948 + 17818905097866076030 + 12738552382738970462 + 17563926324335897738 + 4202646357611763900 + 15610565280136198449 + 10838252318642422779 + 5751812652980752916 + 7707893267971477521 + 3438132572842590888 + 4398435217304672327 + 16731071122206732352 + 14177732933684507045 + 11253425265272496890 + 10504550207674021349 + 559444543901764141 + 11935728146105165543 + 12065888561217960564 + 6715879368372938878 + 7776591968702898575 + 2777892297442503589 + 8393596575808532785 + 10827174201281750732 + 10132611551939750451 + 4013040436104214123 + 9728946780972020798 + 11717397340205329286 + 14855633993462500175 + 8160551898753346851 + 14084807108588385469 + 1567279422163562706 + 11283114109075124863 + 15702454923950617920 + 12874674570724606944 + 15807685566348425479 + 3600542442602250880 + 13860794123126236651 + 18127658097486831628 + 17406633693057974802 + 13089463694395076394 + 994516662000214978 + 3948482519103438457 + 15637130163941710188 + 13258085859046266385 + 13770398394010134258 + 14731955952766459464 + 13812131625811308456 + 7990208979722858489 + 5555587803554688408 + 7912791757010083995 + 1356424559030639541 + 338226593499300764 + 7399798178166915827 + 7949892765528318736 + 8327946347768554799 + 7131733815895456735 + 10561337352372529567 + 9886755048693748436 + 12342420998458234222 + 18272185012221482515 + 2450975714124769815 + 15240240319819211743 + 1276547427350682222 + 11425318910829596551 + 113278865471929396 + 1085778659240049187 + 6576126420121055333 + 10994288847555394400 + 5735659066908006737 + 8776603742348551719 + 2782442786661054022 + 7547596557460541818 + 14971259749014482802 + 8247866334026378787 + 3159320118598665492 + 3153516205698038954 + 13242801524675246308 + 910573127268711400 + 14517389832786916398 + 313860173542898909 + 8422667745160081352 + 13672451047528317787 + 12249368813504890142 + 10103471637433797622 + 13266998533234084192 + 15464861287619024271 + 11486649941824635081 + 9796419365933784848 + 9312270706782964358 + 18285230460419430173 + 11081838169631481396 + 16261865581520769875 + 2071108473059438072 + 9962909113398157430 + 17437177570347930858 + 10076384205656142997 + 7404421492872539392 + 11992094620567828368 + 12947127439459936277 + 15755912943951857513 + 12710691931457120204 + 3965665105313955005 + 10321083169710718356 + 12859317438659787132 + 7827191266905579959 + 8841516927274164065 + 16904399358292801416 + 3103350280716949907 + 4010630207948740390 + 2637559525125777296 + 14963256201043248293 + 14693644106751253929 + 2057594095369183668 + 3690842488290627212 + 9506382959032264239 + 11974663545756131010 + 8323295925319611964 + 14519244833595374159 + 3847204941706060264 + 5536830860483730687 + 12896208588653174561 + 17013735695687392678 + 8346350983856995145 + 17820067124124915269 + 12123170406567551327 + 10124657217191790186 + 634502160289894313 + 7550978471003164232 + 4094776230922722828 + 17188475887859731498 + 7594742709221962116 + 1039590662170898718 + 1271331939192015400 + 11408338179751728497 + 1607474487644489413 + 3635015549777668420 + 14899862472375305307 + 15363547124080264 + 15886171035138107522 + 15453352049913274317 + 11694693379283692291 + 2132069305255467439 + 16870569243420438869 + 18234717839127666423 + 18285965981560479753 + 18111270940626227162 + 14104907628149322697 + 15158769583813602260 + 4900158007465310288 + 16059245221521486098 + 8835079774250545313 + 1959590564828004787 + 15456698581616030214 + 15055913423841403882 + 1655448446099140352 + 4108768096812565019 + 9724548332660252576 + 12373231212148260977 + 15424235784251394897 + 4531799232001851247 + 15984062460192617134 + 11272935587233366998 + 8853864046153834919 + 7146639226724470809 + 2984718311776160456 + 15070396665303514562 + 15367401321295747433 + 12387077815185644578 + 9657244021094071412 + 4461836674847507969 + 4180167270242856127 + 18241195421625251300 + 4105066547526874364 + 12022413181605983622 + 16103562972557811098 + 6659040731699663714 + 10870725420173034865 + 1212681754951069008 + 18025860074680620716 + 5906131488358401977 + 71957734665699424 + 11567540804976982784 + 2140481550254395548 + 9944458069371005838 + 11030757435188357389 + 12990280364598539881 + 3565028159946373568 + 1334051976087420715 + 9006290438966883915 + 965838808964037972 + 4164717558437863614 + 14341871877963814151 + 3485082034215152475 + 12546231594237180069 + 10587484162454639860 + 13614240923783466076 + 3504259420033640996 + 1167824288424891076 + 5546092811808623604 + 937208460419065008 + 1925902397820921919 + 17534477297797308709 + 7090952359397365826 + 16982113483193862176 + 4294602117745642388 + 4131795503687367484 + 1691133646233315032 + 3921624376008763574 + 14334692677173758151 + 14958388812399917866 + 5414021284167547309 + 53111008964741545 + 15460842510158605442 + 18186696881224426947 + 4787514778396839041 + 11884379039943607693 + 7626776604732729889 + 12268202217349925844 + 17230974352098363483 + 16020272298836028903 + 4327732613709196145 + 3573282460319585686 + 6947068702518677381 + 7761123522498093357 + 16568055532312358864 + 8425074468755982712 + 2148470338733320074 + 12246256392907011670 + 1963158579973938344 + 14502857040825842122 + 15515812806640818639 + 10485826424841515738 + 5681452801018167217 + 982745659416036093 + 11735153083848110714 + 6818870286431341260 + 17961429884980870492 + 18029995955319183851 + 7605878799974366109 + 6604944111527816208 + 14781747067457983180 + 1247318105229658450 + 9127209326713543172 + 1506714000539518770 + 3075274491283773496 + 16076599434080024881 + 7642153328316556712 + 11857141806684495137 + 16118183247047629884 + 9526085861565373767 + 8687867772019296640 + 3233333886204900284 + 18167925477273144501 + 2772784027860017033 + 17030353793293721352 + 6978860470841283656 + 1597597641706181670 + 10477988387528564444 + 4929379273223932205 + 16051022509003727042 + 16289045876882967052 + 10032359947712426942 + 7276616997079239490 + 613431939712187006 + 17272077609007089864 + 3319440480636294292 + 14688147951886244835 + 9650287268626680316 + 14327878174987249796 + 16752637269458948196 + 11501126535203194976 + 2317659836353362247 + 17704851477043430102 + 504409360458771772 + 7933019655221080648 + 8511011861368929016 + 9777971681473367131 + 11959568219357422200 + 17611114855351085553 + 5196608542564207124 + 15869889390963813253 + 4743008911590199952 + 4657725142705044680 + 16847406279944882428 + 1236122570182666922 + 4972486929448268837 + 7782177498389426561 + 13383350700086680433 + 6892227676341131892 + 18393780754913436199 + 15366525749877629247 + 6737961354939730160 + 16183958835289748403 + 11960416625298078954 + 423579361091429368 + 1618860279892545620 + 17471930208520045762 + 15348486898381944494 + 15649921972451800223 + 17454920248989374713 + 368641911983013333 + 1471190651490855721 + 4337963864587114984 + 243526101784923158 + 9751678209577508995 + 16846034479237475904 + 9680194725841310016 + 7550399273655542013 + 8285415011050684165 + 14233293819653694114 + 16535070254851058073 + 13617704796229472564 + 9766478673887995247 + 17124001308840712861 + 251690693260949897 + 3314718624207989813 + 5281985406429705176 + 12597020231134382987 + 9782905438451074513 + 17355935286801936580 + 16017265418145351544 + 12747300411223579434 + 711321098380648732 + 4863398326887814242 + 9210599065733202346 + 16573571549449943316 + 7249556861078243366 + 1910858652205186599 + 5250569912794257866 + 1039347345860769292 + 7199861398663786870 + 2011759111183301182 + 7740338971964919436 + 4317321564741068438 + 2074938441103773191 + 4419379448455177799 + 667245781545830005 + 2540729041474268608 + 4208158592511586861 + 14063912840669258147 + 17719062053917364410 + 17312004689762000220 + 5232089336482481146 + 6604861290263336223 + 123590523944275513 + 12406009365553186504 + 2469889520880869100 + 3858879346150415915 + 16579466621405269693 + 17983576308245389640 + 16262453522113220072 + 1693882361212735052 + 16587246742152088064 + 2508682579764432850 + 4832330462517197196 + 16278756989208823310 + 758246325560768291 + 5306018193130314356 + 11422740843545529284 + 5128690802144714991 + 10924050433941265694 + 1632469577207175962 + 15571890758353614494 + 11113091412206882631 + 6622826861857398684 + 13433758488897905996 + 1133813309861010178 + 13876281389188668628 + 1496844208448379722 + 14035655260784191760 + 7632503394576793159 + 1996405771784781732 + 12273423142638581190 + 6447652678883147921 + 13640817613751710072 + 1888485066898739132 + 5093501448291383046 + 3952069885294190767 + 12940205664340693565 + 7908501217890455204 + 7520597156587673184 + 767767061119854846 + 11898339525032033768 + 12123850033636734886 + 8765200496337861713 + 1679221168964040643 + 13159931486887302320 + 17545841896314162371 + 18407555266226246062 + 7085618836503982785 + 13508631139981686746 + 4776686543043414990 + 10257766090624152320 + 15478360341159100954 + 10866814433432653400 + 1266181251575719938 + 9874832045439102043 + 12670121444072556821 + 14415048846072178379 + 8576790772504445299 + 10398215560057959490 + 13793955191481823072 + 7705926180284189770 + 764675236710303754 + 13230695744109747833 + 17117739322459402141 + 14251152644700291132 + 17925924700753351551 + 8014403157810821343 + 7663656250015373447 + 15986729484496988211 + 16993995513943202776 + 8163314895115788322 + 13893840939053554287 + 10385068827404185981 + 5193058723474298989 + 14597908901451293948 + 10460419535139862480 + 10967483515691179220 + 17460876096361245793 + 10814764302177548819 + 2624558864665368841 + 466468485564559014 + 14252181633448897528 + 12904479801311122937 + 4036418914598247369 + 5025738328865459682 + 17804665164863226244 + 5615667923431706137 + 13088986081549949361 + 12877985488322567902 + 1146903302020974453 + 4860964478930959825 + 12085875644594601176 + 13116092602275838627 + 8321274857970877952 + 9385482918563989479 + 17912902819951822195 + 5646770101675234126 + 15573306932819096276 + 4758405545863482220 + 7216157059662308903 + 5502158873643706387 + 15108922535060768617 + 5491558681199354700 + 18350474900665525109 + 15190037752693953036 + 17411353686479605512 + 12258897571827613118 + 14119755389067763941 + 4753679948540725779 + 8337818043130916921 + 16551317428984357254 + 2707004360768664818 + 6465721769240061502 + 8132094879699321965 + 6795404745535059397 + 14462718767334455805 + 6113724564439377093 + 13146256235160797215 + 1241192493374572077 + 13018575274399579234 + 7343738444176544790 + 9599152601268642801 + 12797359411031704797 + 10175860064669295644 + 7663976226311323290 + 16045148899254928993 + 6587365537689360100 + 14926181263652885738 + 11057016571685063825 + 11574619060238502434 + 894827397848194018 + 2375192706578325948 + 9592667208480817039 + 11632186791661274942 + 11743339922943929062 + 1499561529162544247 + 948270756520396313 + 6841514893615804338 + 16895894165983749133 + 12760113486386773161 + 10316393095044339846 + 2808149728906376039 + 4577729166617804641 + 15878203270828380512 + 12159856899460557607 + 16401966270784925039 + 13703235828125908965 + 8217866906322789475 + 9211632184155679570 + 11111651373357368944 + 17640788901914951619 + 6792822383765862073 + 18061924544116450169 + 9436378252498679448 + 4458722208509575818 + 6034406588632393305 + 14816646944272162649 + 138194924977623443 + 15596320980990824012 + 12301195492059219541 + 13834750789127766439 + 1204524306023886050 + 15363133096392070863 + 12408063731949899568 + 3381237095788978962 + 15035708039266375057 + 15520167763123795952 + 502309266672748278 + 15789773558864482061 + 6582341717764178990 + 1974176855187298409 + 11915423680871385683 + 14010664758385690810 + 17773687691883612951 + 574035821170072339 + 13400956510153680374 + 12725170784205124486 + 2395650900435004906 + 6693438811263003514 + 6721658043245960168 + 1314475145545084692 + 15249277904583121423 + 10495353025269458671 + 15159745149031414970 + 13549997004086666295 + 6163157753455540046 + 16795271496376618129 + 3446235441568115468 + 8770576388012584819 + 10901260234870783004 + 11905811876774412952 + 4859007370590074533 + 18321360744669247296 + 149593614613851662 + 16684443475213896998 + 6689023493705820156 + 12860911923327197762 + 15812481719359863190 + 11128280823149323477 + 3255706166971391764 + 14786367190455492567 + 3785824395222472965 + 10823542274683292235 + 13249656743621513144 + 15793380991656641504 + 732122365848928568 + 14939694193888499370 + 13144162389086683969 + 9429109771226952298 + 17729816496430879681 + 11077661704251967142 + 2621304554568702565 + 9385059778434368421 + 14203382899255954515 + 6012093655693704712 + 16207819942499785577 + 4502090935915637323 + 9495016594161345114 + 12636017273154187901 + 2897494169871979697 + 14185923395245048393 + 5095150295067142356 + 10214290467161590775 + 5190086744227302393 + 17212599803849087231 + 465816217519226066 + 1750147651096625822 + 5994193402555014414 + 5264719330888591226 + 3401857805039875323 + 1445654113107514901 + 14138990935534866273 + 15668896640647016294 + 5737728875218183219 + 17249084056589068385 + 18254563909328505321 + 15750037654276632904 + 11121974791251290772 + 5978888750754466241 + 8889260741872114810 + 4747860697660210768 + 17048526165086336820 + 4054112028194783106 + 6938467658411184333 + 12982152377953076112 + 4196853978726079783 + 2175349381112603518 + 3784051276405904862 + 221248721105895659 + 5647694285588324749 + 4200225056987130659 + 2695671588831458243 + 8839772794713247554 + 12212976363431249396 + 3489732583322888353 + 5547221163310471790 + 1827588699821069081 + 17866135417617074562 + 3181922582949110124 + 10286725530397007766 + 1118172031856817028 + 7346319743713496484 + 4748038856156779402 + 5847596696702621933 + 2531357294975150270 + 522839523479360477 + 13896540934619858178 + 11076458389499186531 + 2752038081986354656 + 15727718023992713164 + 4728018142447912884 + 15518664270597186063 + 2134693874838389131 + 10228895972860682916 + 11459821992144236506 + 7869788186870232011 + 14970890481879241566 + 8756941675290048218 + 10869254214162790925 + 4377255505598592686 + 1216538299119732450 + 10132312819630456949 + 11863534461176728537 + 18286057519007701831 + 1677069787769523732 + 1104163571263092446 + 1121928605306773068 + 12430223770234903720 + 9840298539596873639 + 12693163215434654919 + 3700880022863857158 + 5276174972450466522 + 12814001426429226986 + 15706239720435893291 + 18042109094879760253 + 10948615188017653951 + 15418100339082629183 + 10069459694679290961 + 17174683563421007447 + 13915750074112662351 + 3093270745496639167 + 11663441915142931092 + 578446115736726622 + 13264473419900147559 + 6667495605547652544 + 3482199446278800566 + 5089686380256313942 + 13030172275929047244 + 8907410104086258772 + 8883694939182477848 + 15317614598641847523 + 2607706018662263347 + 3732632051731565844 + 13283514949726179699 + 18262833696957702227 + 10937980486533216958 + 14059491137820776060 + 5958511359126346725 + 14810779933281216412 + 4875580198187922563 + 1716469496271165568 + 233421718396168540 + 9531475641935766997 + 2333078153872791210 + 16578820316563458547 + 11250701909169607395 + 15651561691176830486 + 15185866523902386864 + 10624677799139518407 + 16535155009388086574 + 6964584063087669942 + 4275162838434199970 + 3346445358120280306 + 11859451888921262770 + 14200620539315128865 + 16942683791977556558 + 16545367021290979448 + 10185105092876908363 + 9032625984754504084 + 10840870981553070388 + 953505980170313013 + 4759681357099865694 + 2563804318228320264 + 18074208644259464665 + 304371792302462860 + 11516390401314470367 + 12318693479523493913 + 2479492432100862863 + 1205835192505990906 + 9385295035871814889 + 11542112419936942421 + 17422237772384476687 + 5139964178785829996 + 8556489727584771937 + 8550553698582458016 + 3340724501593222760 + 15323697355437391430 + 3099300620123954197 + 10529159653326641864 + 7504508221263058116 + 16757640508849246008 + + diff --git a/Sources/CompletionScoringTestSupport/RepeatableRandomNumberGenerator.swift b/Sources/CompletionScoringTestSupport/RepeatableRandomNumberGenerator.swift new file mode 100644 index 000000000..15d091654 --- /dev/null +++ b/Sources/CompletionScoringTestSupport/RepeatableRandomNumberGenerator.swift @@ -0,0 +1,116 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import CompletionScoring +import Foundation + +package struct RepeatableRandomNumberGenerator: RandomNumberGenerator { + private let seed: [UInt64] + private static let startIndex = (0, 1, 2, 3) + private var nextIndex = Self.startIndex + + package init() { + self.seed = try! PropertyListDecoder().decode( + [UInt64].self, + from: loadTestResource(name: "RandomSeed", withExtension: "plist") + ) + } + + @discardableResult + func increment(value: inout Int, range: Range) -> Bool { + value += 1 + let wrapped = (value == range.upperBound) + if wrapped { + value = range.lowerBound + } + return wrapped + } + + mutating func advance() { + // This iterates through "K choose N" or "seed.count choose 4" unique combinations of 4 seed indexes. Given 1024 values in seed, that produces 45,545,029,376 unique combination of the values. + if increment(value: &nextIndex.3, range: (nextIndex.2 + 1)..<(seed.count - 0)) { + if increment(value: &nextIndex.2, range: (nextIndex.1 + 1)..<(seed.count - 1)) { + if increment(value: &nextIndex.1, range: (nextIndex.0 + 1)..<(seed.count - 2)) { + if increment(value: &nextIndex.0, range: 0..<(seed.count - 3)) { + nextIndex = Self.startIndex + return + } + nextIndex.1 = (nextIndex.0 + 1) + } + nextIndex.2 = (nextIndex.1 + 1) + } + nextIndex.3 = (nextIndex.2 + 1) + } + } + + package mutating func next() -> UInt64 { + let result = seed[nextIndex.0] ^ seed[nextIndex.1] ^ seed[nextIndex.2] ^ seed[nextIndex.3] + advance() + return result + } + + static func generateSeed() { + let numbers: [UInt64] = (0..<1024).map { _ in + let lo = UInt64(arc4random()) + let hi = UInt64(arc4random()) + return (hi << 32) | lo + } + let header = + """ + + + + + """ + let body = numbers.map { number in " \(number)" } + let footer = + """ + + + """ + + print(([header] + body + [footer]).joined(separator: "\n")) + } + + package mutating func randomLowercaseASCIIString(lengthRange: ClosedRange) -> String { + let length = lengthRange.randomElement(using: &self) + let utf8Bytes = (0.., + lengthRange: ClosedRange + ) -> [String] { + let count = countRange.randomElement(using: &self) + var strings: [String] = [] + for _ in (0.. Data { + let file = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() + .appendingPathComponent("\(name).\(ext)") + return try Data(contentsOf: file) +} + +extension ClosedRange { + func randomElement(using randomness: inout Generator) -> Element { + return randomElement(using: &randomness)! // Closed ranges always have a value + } +} diff --git a/Sources/CompletionScoringTestSupport/SymbolGenerator.swift b/Sources/CompletionScoringTestSupport/SymbolGenerator.swift new file mode 100644 index 000000000..8e64e567f --- /dev/null +++ b/Sources/CompletionScoringTestSupport/SymbolGenerator.swift @@ -0,0 +1,201 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import CompletionScoring +import Foundation + +package struct SymbolGenerator: Sendable { + package static let shared = Self() + + struct TermTable { + struct Entry: Codable { + var word: String + var count: Int + } + private var replicatedTerms: [String] + + init() throws { + let entries = try JSONDecoder().decode( + [Entry].self, + from: loadTestResource(name: "CommonFunctionTerms", withExtension: "json") + ) + let repeatedWords: [[String]] = entries.map { entry in + var word = entry.word + // Make the first letter lowercase if the word isn't something like 'URL'. + if word.count > 1 { + let first = entry.word.startIndex + let second = entry.word.index(after: first) + if word[first].isUppercase && !word[second].isUppercase { + let head = word[first].lowercased() + let tail = word.dropFirst(1) + word = head + tail + } + } + return Array(repeating: word, count: entry.count) + } + replicatedTerms = Array(repeatedWords.joined()) + } + + func randomTerm(using randomness: inout RepeatableRandomNumberGenerator) -> String { + replicatedTerms.randomElement(using: &randomness)! + } + } + + let termTable = try! TermTable() + let segmentCountWeights = WeightedChoices([ + (0.5, 1), + (0.375, 2), + (0.125, 3), + ]) + + package func randomSegment( + using randomness: inout RepeatableRandomNumberGenerator, + capitalizeFirstTerm: Bool + ) -> String { + let count = segmentCountWeights.select(using: &randomness) + return (0.. 0 || capitalizeFirstTerm + return capitalize ? term.capitalized : term + }.joined() + } + + package func randomType(using randomness: inout RepeatableRandomNumberGenerator) -> String { + randomSegment(using: &randomness, capitalizeFirstTerm: true) + } + + let argumentCountWeights = WeightedChoices([ + (0.333, 0), + (0.333, 1), + (0.250, 2), + (0.083, 3), + ]) + + package struct Function { + var baseName: String + var arguments: [Argument] + + package var filterText: String { + let argPattern: String + if arguments.hasContent { + argPattern = arguments.map { argument in + (argument.label ?? "") + ":" + }.joined() + } else { + argPattern = "" + } + return baseName + "(" + argPattern + ")" + } + + package var displayText: String { + let argPattern: String + if arguments.hasContent { + argPattern = arguments.map { argument in + (argument.label ?? "_") + ": " + argument.type + }.joined(separator: ", ") + } else { + argPattern = "" + } + return baseName + "(" + argPattern + ")" + } + } + + struct Argument { + var label: String? + var type: String + } + + let argumentLabeledWeights = WeightedChoices([ + (31 / 32.0, true), + (01 / 32.0, false), + ]) + + func randomArgument(using randomness: inout RepeatableRandomNumberGenerator) -> Argument { + let labeled = argumentLabeledWeights.select(using: &randomness) + let label = labeled ? randomSegment(using: &randomness, capitalizeFirstTerm: false) : nil + return Argument(label: label, type: randomType(using: &randomness)) + } + + package func randomFunction(using randomness: inout RepeatableRandomNumberGenerator) -> Function { + let argCount = argumentCountWeights.select(using: &randomness) + return Function( + baseName: randomSegment(using: &randomness, capitalizeFirstTerm: false), + arguments: Array(count: argCount) { + randomArgument(using: &randomness) + } + ) + } + + let initializerCounts = WeightedChoices([ + (32 / 64.0, 1), + (16 / 64.0, 2), + (8 / 64.0, 3), + (4 / 64.0, 4), + (2 / 64.0, 5), + (1 / 64.0, 6), + (1 / 64.0, 0), + ]) + + let initializerArgumentCounts = WeightedChoices([ + (512 / 1024.0, 1), + (256 / 1024.0, 2), + (128 / 1024.0, 3), + (64 / 1024.0, 4), + (58 / 1024.0, 0), + (4 / 1024.0, 16), + (2 / 1024.0, 32), + ]) + + package func randomInitializers( + typeName: String, + using randomness: inout RepeatableRandomNumberGenerator + ) -> [Function] { + let initializerCount = initializerCounts.select(using: &randomness) + return Array(count: initializerCount) { + let argumentCount = initializerArgumentCounts.select(using: &randomness) + let arguments: [Argument] = Array(count: argumentCount) { + randomArgument(using: &randomness) + } + return Function(baseName: typeName, arguments: arguments) + } + } + + let capitalizedPatternWeights = WeightedChoices([ + (7 / 8.0, false), + (1 / 8.0, true), + ]) + + package func randomPatternText( + lengthRange: Range, + using randomness: inout RepeatableRandomNumberGenerator + ) -> String { + var text = "" + while text.count < lengthRange.upperBound { + text = randomSegment( + using: &randomness, + capitalizeFirstTerm: capitalizedPatternWeights.select(using: &randomness) + ) + } + let length = lengthRange.randomElement(using: &randomness) ?? 0 + return String(text.prefix(length)) + } + + func randomAPIs( + functionCount: Int, + typeCount: Int, + using randomness: inout RepeatableRandomNumberGenerator + ) -> [String] { + let functions = (0..=6) +package import CompletionScoring +#else +import CompletionScoring +#endif + +@inline(never) +package func drain(_ value: T) {} + +func duration(of body: () -> ()) -> TimeInterval { + let start = ProcessInfo.processInfo.systemUptime + body() + return ProcessInfo.processInfo.systemUptime - start +} + +extension RandomNumberGenerator { + mutating func nextBool() -> Bool { + (next() & 0x01 == 0x01) + } +} + +package func withEachPermutation(_ a: T, _ b: T, body: (T, T) -> ()) { + body(a, b) + body(b, a) +} + +extension XCTestCase { + private func heatUp() { + var integers = 1024 + var elapsed = 0.0 + while elapsed < 1.0 { + elapsed += duration { + let integers = Array(count: integers) { + arc4random() + } + DispatchQueue.concurrentPerform(iterations: 128) { _ in + integers.withUnsafeBytes { bytes in + var hasher = Hasher() + hasher.combine(bytes: bytes) + drain(hasher.finalize()) + } + } + } + integers *= 2 + } + } + + private func coolDown() { + Thread.sleep(forTimeInterval: 2.0) + } + + func induceThermalRange(_ range: ClosedRange) { + var temperature: Int + repeat { + temperature = ProcessInfo.processInfo.thermalLevel() + if temperature < range.lowerBound { + print("Too Cold: \(temperature)") + heatUp() + } else if temperature > range.upperBound { + print("Too Hot: \(temperature)") + coolDown() + } + } while !range.contains(temperature) + } + + private static let targetThermalRange: ClosedRange? = { + if ProcessInfo.processInfo.environment["SOURCEKIT_LSP_PERFORMANCE_MEASUREMENTS_ENABLE_THERMAL_THROTTLING"] + != nil + { + // This range is arbitrary. All that matters is that the same values are used on the baseline and the branch. + let target = + ProcessInfo.processInfo.environment["SOURCEKIT_LSP_PERFORMANCE_MEASUREMENTS_THERMAL_TARGET"].flatMap( + Int.init + ) + ?? 75 + let variance = + ProcessInfo.processInfo.environment["SOURCEKIT_LSP_PERFORMANCE_MEASUREMENTS_THERMAL_VARIANCE"].flatMap( + Int.init + ) + ?? 5 + return (target - variance)...(target + variance) + } else { + return nil + } + }() + + private static let measurementsLogFile: String? = { + UserDefaults.standard.string(forKey: "TestMeasurementLogPath") + }() + + static let printBeginingOfLog = AtomicBool(initialValue: true) + + private static func openPerformanceLog() throws -> FileHandle? { + try measurementsLogFile.map { path in + if !FileManager.default.fileExists(atPath: path) { + try FileManager.default.createDirectory( + at: URL(fileURLWithPath: path).deletingLastPathComponent(), + withIntermediateDirectories: true + ) + FileManager.default.createFile(atPath: path, contents: Data()) + } + let logFD = try FileHandle(forWritingAtPath: path).unwrap(orThrow: "Opening \(path) failed") + try logFD.seekToEnd() + if printBeginingOfLog.value { + try logFD.print("========= \(Date().description(with: .current)) =========") + printBeginingOfLog.value = false + } + return logFD + } + } + + private var testName: String? { + self.invocation?.selector.description.dropPrefix("test").dropSuffix("AndReturnError:") + } + + private var testDisplayName: String { + return testName ?? "(Unnamed Test)" + } + + func tryOrFailTest(_ expression: @autoclosure () throws -> R?, message: String) -> R? { + do { + return try expression() + } catch { + XCTFail("\(message): \(error)") + return nil + } + } + + /// Run `body()` `iterations`, gathering timing stats, and print them. + /// In between runs, coax for the machine into an arbitrary but consistent thermal state by either sleeping or doing + /// pointless work so that results are more comparable run to run, no matter else is happening on the machine. + package func gaugeTiming(iterations: Int = 1, _ body: () -> ()) { + let logFD = tryOrFailTest(try Self.openPerformanceLog(), message: "Failed to open performance log") + var timings = Timings() + for iteration in 0.. Int { + var thermalLevel: UInt64 = 0 + var size: size_t = MemoryLayout.size + sysctlbyname("machdep.xcpm.cpu_thermal_level", &thermalLevel, &size, nil, 0) + return Int(thermalLevel) + } +} + +extension String { + fileprivate func dropSuffix(_ suffix: String) -> String { + if hasSuffix(suffix) { + return String(dropLast(suffix.count)) + } + return self + } + + fileprivate func dropPrefix(_ prefix: String) -> String { + if hasPrefix(prefix) { + return String(dropFirst(prefix.count)) + } + return self + } + + package func allocateCopyOfUTF8Buffer() -> CompletionScoring.Pattern.UTF8Bytes { + withUncachedUTF8Bytes { utf8Buffer in + UnsafeBufferPointer.allocate(copyOf: utf8Buffer) + } + } +} + +extension FileHandle { + func write(_ text: String) throws { + try text.withUncachedUTF8Bytes { bytes in + try write(contentsOf: bytes) + } + } + + func print(_ text: String) throws { + try write(text) + try write("\n") + } +} + +extension Double { + func format(_ specifier: StringLiteralType) -> String { + String(format: specifier, self) + } +} + +extension Int { + func format(_ specifier: StringLiteralType) -> String { + String(format: specifier, self) + } +} + +extension CandidateBatch { + package init(symbols: [String]) { + self.init(candidates: symbols, contentType: .codeCompletionSymbol) + } +} diff --git a/Sources/CompletionScoringTestSupport/Timings.swift b/Sources/CompletionScoringTestSupport/Timings.swift new file mode 100644 index 000000000..3d47e8551 --- /dev/null +++ b/Sources/CompletionScoringTestSupport/Timings.swift @@ -0,0 +1,117 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import CompletionScoring +import Foundation + +package struct Timings { + package struct Stats { + package private(set) var min: Double + package private(set) var max: Double + private var total: Double + private var count: Int + + fileprivate init(initialValue: Double) { + total = initialValue + min = initialValue + max = initialValue + count = 1 + } + + fileprivate mutating func append(_ value: Double) { + count += 1 + total += value + min = Swift.min(min, value) + max = Swift.max(max, value) + } + + var average: Double { + total / Double(count) + } + } + + package private(set) var stats: Stats? = nil + private(set) var values: [Double] = [] + + package init(_ values: [Double] = []) { + for value in values { + append(value) + } + } + + private var hasVariation: Bool { + return values.count >= 2 + } + + package var meanAverageDeviation: Double { + if let stats = stats, hasVariation { + var sumOfDiviations = 0.0 + for value in values { + sumOfDiviations += abs(value - stats.average) + } + return sumOfDiviations / Double(values.count) + } else { + return 0 + } + } + + package var standardDeviation: Double { + if let stats = stats, hasVariation { + var sumOfSquares = 0.0 + for value in values { + let deviation = (value - stats.average) + sumOfSquares += deviation * deviation + } + let variance = sumOfSquares / Double(values.count - 1) + return sqrt(variance) + } else { + return 0 + } + } + + package var standardError: Double { + if hasVariation { + return standardDeviation / sqrt(Double(values.count)) + } else { + return 0 + } + } + + /// There's 95% confidence that the true mean is with this distance from the sampled mean. + var confidenceOfMean_95Percent: Double { + if stats != nil { + return 1.96 * standardError + } + return 0 + } + + @discardableResult + mutating func append(_ value: Double) -> Stats { + values.append(value) + stats.mutateWrappedValue { stats in + stats.append(value) + } + return stats.lazyInitialize { + Stats(initialValue: value) + } + } +} + +extension Optional { + mutating func mutateWrappedValue(mutator: (inout Wrapped) -> ()) { + if var wrapped = self { + self = nil // Avoid COW for clients. + mutator(&wrapped) + self = wrapped + } + } +} diff --git a/Sources/CompletionScoringTestSupport/WeightedChoices.swift b/Sources/CompletionScoringTestSupport/WeightedChoices.swift new file mode 100644 index 000000000..b3a534453 --- /dev/null +++ b/Sources/CompletionScoringTestSupport/WeightedChoices.swift @@ -0,0 +1,34 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import CompletionScoring +import Foundation + +package struct WeightedChoices: Sendable { + package typealias WeightedChoice = (likelihood: Double, value: T) + private var choices: [T] = [] + + package init(_ choices: [WeightedChoice]) { + precondition(choices.hasContent) + let smallest = choices.map(\.likelihood).min()! + let samples = 1.0 / (smallest / 2.0) + 1 + for choice in choices { + precondition(choice.likelihood > 0) + precondition(choice.likelihood <= 1.0) + self.choices.append(contentsOf: Array(repeating: choice.value, count: Int(choice.likelihood * samples))) + } + } + + package func select(using randomness: inout RepeatableRandomNumberGenerator) -> T { + choices.randomElement(using: &randomness)! + } +} diff --git a/Sources/Csourcekitd/include/CodeCompletionSwiftInterop.h b/Sources/Csourcekitd/include/CodeCompletionSwiftInterop.h new file mode 100644 index 000000000..8492145ec --- /dev/null +++ b/Sources/Csourcekitd/include/CodeCompletionSwiftInterop.h @@ -0,0 +1,408 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +#ifndef SWIFT_C_CODE_COMPLETION_H +#define SWIFT_C_CODE_COMPLETION_H + +#include +#include +#include + +/// Global state across completions including compiler instance caching. +typedef void *swiftide_api_connection_t; + +/// Opaque completion item handle, used to retrieve additional information that +/// may be more expensive to compute. +typedef void *swiftide_api_completion_item_t; + +typedef enum { + SWIFTIDE_COMPLETION_KIND_NONE = 0, + SWIFTIDE_COMPLETION_KIND_IMPORT = 1, + SWIFTIDE_COMPLETION_KIND_UNRESOLVEDMEMBER = 2, + SWIFTIDE_COMPLETION_KIND_DOTEXPR = 3, + SWIFTIDE_COMPLETION_KIND_STMTOREXPR = 4, + SWIFTIDE_COMPLETION_KIND_POSTFIXEXPRBEGINNING = 5, + SWIFTIDE_COMPLETION_KIND_POSTFIXEXPR = 6, + /* obsoleted */SWIFTIDE_COMPLETION_KIND_POSTFIXEXPRPAREN = 7, + SWIFTIDE_COMPLETION_KIND_KEYPATHEXPROBJC = 8, + SWIFTIDE_COMPLETION_KIND_KEYPATHEXPRSWIFT = 9, + SWIFTIDE_COMPLETION_KIND_TYPEDECLRESULTBEGINNING = 10, + SWIFTIDE_COMPLETION_KIND_TYPESIMPLEBEGINNING = 11, + SWIFTIDE_COMPLETION_KIND_TYPEIDENTIFIERWITHDOT = 12, + SWIFTIDE_COMPLETION_KIND_TYPEIDENTIFIERWITHOUTDOT = 13, + SWIFTIDE_COMPLETION_KIND_CASESTMTKEYWORD = 14, + SWIFTIDE_COMPLETION_KIND_CASESTMTBEGINNING = 15, + SWIFTIDE_COMPLETION_KIND_NOMINALMEMBERBEGINNING = 16, + SWIFTIDE_COMPLETION_KIND_ACCESSORBEGINNING = 17, + SWIFTIDE_COMPLETION_KIND_ATTRIBUTEBEGIN = 18, + SWIFTIDE_COMPLETION_KIND_ATTRIBUTEDECLPAREN = 19, + SWIFTIDE_COMPLETION_KIND_POUNDAVAILABLEPLATFORM = 20, + SWIFTIDE_COMPLETION_KIND_CALLARG = 21, + SWIFTIDE_COMPLETION_KIND_LABELEDTRAILINGCLOSURE = 22, + SWIFTIDE_COMPLETION_KIND_RETURNSTMTEXPR = 23, + SWIFTIDE_COMPLETION_KIND_YIELDSTMTEXPR = 24, + SWIFTIDE_COMPLETION_KIND_FOREACHSEQUENCE = 25, + SWIFTIDE_COMPLETION_KIND_AFTERPOUNDEXPR = 26, + SWIFTIDE_COMPLETION_KIND_AFTERPOUNDDIRECTIVE = 27, + SWIFTIDE_COMPLETION_KIND_PLATFORMCONDITON = 28, + SWIFTIDE_COMPLETION_KIND_AFTERIFSTMTELSE = 29, + SWIFTIDE_COMPLETION_KIND_GENERICREQUIREMENT = 30, + SWIFTIDE_COMPLETION_KIND_PRECEDENCEGROUP = 31, + SWIFTIDE_COMPLETION_KIND_STMTLABEL = 32, + SWIFTIDE_COMPLETION_KIND_EFFECTSSPECIFIER = 33, + SWIFTIDE_COMPLETION_KIND_FOREACHPATTERNBEGINNING = 34, + SWIFTIDE_COMPLETION_KIND_TYPEATTRBEGINNING = 35, + SWIFTIDE_COMPLETION_KIND_OPTIONALBINDING = 36, + SWIFTIDE_COMPLETION_KIND_FOREACHKWIN = 37, + SWIFTIDE_COMPLETION_KIND_WITHOUTCONSTRAINTTYPE = 38, + SWIFTIDE_COMPLETION_KIND_THENSTMTEXPR = 39, + SWIFTIDE_COMPLETION_KIND_TYPEBEGINNING = 40, + SWIFTIDE_COMPLETION_KIND_TYPESIMPLEORCOMPOSITION = 41, + SWIFTIDE_COMPLETION_KIND_TYPEPOSSIBLEFUNCTIONPARAMBEGINNING = 42, + SWIFTIDE_COMPLETION_KIND_TYPEATTRINHERITANCEBEGINNING = 43, +} swiftide_api_completion_kind_t; + +typedef enum { + SWIFTIDE_COMPLETION_ITEM_KIND_DECLARATION = 0, + SWIFTIDE_COMPLETION_ITEM_KIND_KEYWORD = 1, + SWIFTIDE_COMPLETION_ITEM_KIND_PATTERN = 2, + SWIFTIDE_COMPLETION_ITEM_KIND_LITERAL = 3, + SWIFTIDE_COMPLETION_ITEM_KIND_BUILTINOPERATOR = 4, +} swiftide_api_completion_item_kind_t; + +typedef enum { + SWIFTIDE_COMPLETION_ITEM_DECL_KIND_MODULE = 0, + SWIFTIDE_COMPLETION_ITEM_DECL_KIND_CLASS = 1, + SWIFTIDE_COMPLETION_ITEM_DECL_KIND_STRUCT = 2, + SWIFTIDE_COMPLETION_ITEM_DECL_KIND_ENUM = 3, + SWIFTIDE_COMPLETION_ITEM_DECL_KIND_ENUMELEMENT = 4, + SWIFTIDE_COMPLETION_ITEM_DECL_KIND_PROTOCOL = 5, + SWIFTIDE_COMPLETION_ITEM_DECL_KIND_ASSOCIATEDTYPE = 6, + SWIFTIDE_COMPLETION_ITEM_DECL_KIND_TYPEALIAS = 7, + SWIFTIDE_COMPLETION_ITEM_DECL_KIND_GENERICTYPEPARAM = 8, + SWIFTIDE_COMPLETION_ITEM_DECL_KIND_CONSTRUCTOR = 9, + SWIFTIDE_COMPLETION_ITEM_DECL_KIND_DESTRUCTOR = 10, + SWIFTIDE_COMPLETION_ITEM_DECL_KIND_SUBSCRIPT = 11, + SWIFTIDE_COMPLETION_ITEM_DECL_KIND_STATICMETHOD = 12, + SWIFTIDE_COMPLETION_ITEM_DECL_KIND_INSTANCEMETHOD = 13, + SWIFTIDE_COMPLETION_ITEM_DECL_KIND_PREFIXOPERATORFUNCTION = 14, + SWIFTIDE_COMPLETION_ITEM_DECL_KIND_POSTFIXOPERATORFUNCTION = 15, + SWIFTIDE_COMPLETION_ITEM_DECL_KIND_INFIXOPERATORFUNCTION = 16, + SWIFTIDE_COMPLETION_ITEM_DECL_KIND_FREEFUNCTION = 17, + SWIFTIDE_COMPLETION_ITEM_DECL_KIND_STATICVAR = 18, + SWIFTIDE_COMPLETION_ITEM_DECL_KIND_INSTANCEVAR = 19, + SWIFTIDE_COMPLETION_ITEM_DECL_KIND_LOCALVAR = 20, + SWIFTIDE_COMPLETION_ITEM_DECL_KIND_GLOBALVAR = 21, + SWIFTIDE_COMPLETION_ITEM_DECL_KIND_PRECEDENCEGROUP = 22, + SWIFTIDE_COMPLETION_ITEM_DECL_KIND_ACTOR = 23, + SWIFTIDE_COMPLETION_ITEM_DECL_KIND_MACRO = 24, +} swiftide_api_completion_item_decl_kind_t; + +typedef enum { + SWIFTIDE_COMPLETION_TYPE_RELATION_NOTAPPLICABLE = 0, + SWIFTIDE_COMPLETION_TYPE_RELATION_UNKNOWN = 1, + SWIFTIDE_COMPLETION_TYPE_RELATION_UNRELATED = 2, + SWIFTIDE_COMPLETION_TYPE_RELATION_INVALID = 3, + SWIFTIDE_COMPLETION_TYPE_RELATION_CONVERTIBLE = 4, + SWIFTIDE_COMPLETION_TYPE_RELATION_IDENTICAL = 5, +} swiftide_api_completion_type_relation_t; + +typedef enum { + SWIFTIDE_COMPLETION_SEMANTIC_CONTEXT_NONE = 0, + /* obsoleted */SWIFTIDE_COMPLETION_SEMANTIC_CONTEXT_EXPRESSIONSPECIFIC = 1, + SWIFTIDE_COMPLETION_SEMANTIC_CONTEXT_LOCAL = 2, + SWIFTIDE_COMPLETION_SEMANTIC_CONTEXT_CURRENTNOMINAL = 3, + SWIFTIDE_COMPLETION_SEMANTIC_CONTEXT_SUPER = 4, + SWIFTIDE_COMPLETION_SEMANTIC_CONTEXT_OUTSIDENOMINAL = 5, + SWIFTIDE_COMPLETION_SEMANTIC_CONTEXT_CURRENTMODULE = 6, + SWIFTIDE_COMPLETION_SEMANTIC_CONTEXT_OTHERMODULE = 7, +} swiftide_api_completion_semantic_context_t; + +typedef enum { + SWIFTIDE_COMPLETION_FLAIR_EXPRESSIONSPECIFIC = 1 << 0, + SWIFTIDE_COMPLETION_FLAIR_SUPERCHAIN = 1 << 1, + SWIFTIDE_COMPLETION_FLAIR_ARGUMENTLABELS = 1 << 2, + SWIFTIDE_COMPLETION_FLAIR_COMMONKEYWORDATCURRENTPOSITION = 1 << 3, + SWIFTIDE_COMPLETION_FLAIR_RAREKEYWORDATCURRENTPOSITION = 1 << 4, + SWIFTIDE_COMPLETION_FLAIR_RARETYPEATCURRENTPOSITION = 1 << 5, + SWIFTIDE_COMPLETION_FLAIR_EXPRESSIONATNONSCRIPTORMAINFILESCOPE = 1 << 6, +} swiftide_api_completion_flair_t; + +typedef enum { + SWIFTIDE_COMPLETION_NOT_RECOMMENDED_NONE = 0, + SWIFTIDE_COMPLETION_NOT_RECOMMENDED_REDUNDANT_IMPORT = 1, + SWIFTIDE_COMPLETION_NOT_RECOMMENDED_DEPRECATED = 2, + SWIFTIDE_COMPLETION_NOT_RECOMMENDED_INVALID_ASYNC_CONTEXT = 3, + SWIFTIDE_COMPLETION_NOT_RECOMMENDED_CROSS_ACTOR_REFERENCE = 4, + SWIFTIDE_COMPLETION_NOT_RECOMMENDED_VARIABLE_USED_IN_OWN_DEFINITION = 5, + SWIFTIDE_COMPLETION_NOT_RECOMMENDED_REDUNDANT_IMPORT_INDIRECT = 6, + SWIFTIDE_COMPLETION_NOT_RECOMMENDED_SOFTDEPRECATED = 7, + SWIFTIDE_COMPLETION_NOT_RECOMMENDED_NON_ASYNC_ALTERNATIVE_USED_IN_ASYNC_CONTEXT = 8, +} swiftide_api_completion_not_recommended_reason_t; + +typedef enum { + SWIFTIDE_COMPLETION_DIAGNOSTIC_SEVERITY_NONE = 0, + SWIFTIDE_COMPLETION_DIAGNOSTIC_SEVERITY_ERROR = 1, + SWIFTIDE_COMPLETION_DIAGNOSTIC_SEVERITY_WARNING = 2, + SWIFTIDE_COMPLETION_DIAGNOSTIC_SEVERITY_REMARK = 3, + SWIFTIDE_COMPLETION_DIAGNOSTIC_SEVERITY_NOTE = 4, +} swiftide_api_completion_diagnostic_severity_t; + +typedef void *swiftide_api_completion_request_t; + +typedef void *swiftide_api_completion_response_t; + +typedef void *swiftide_api_fuzzy_match_pattern_t; + +typedef void *swiftide_api_cache_invalidation_options_t; + +/// swiftide equivalent of sourcekitd_request_handle_t +typedef const void *swiftide_api_request_handle_t; + + +typedef struct { + _Nonnull swiftide_api_connection_t (*_Nonnull connection_create_with_inspection_instance)( + void *_Null_unspecified opqueSwiftIDEInspectionInstance + ); + + void (*_Nonnull connection_dispose)( + _Null_unspecified swiftide_api_connection_t + ); + + void (*_Nonnull connection_mark_cached_compiler_instance_should_be_invalidated)( + _Null_unspecified swiftide_api_connection_t, + _Null_unspecified swiftide_api_cache_invalidation_options_t + ); + + /// Override the contents of the file \p path with \p contents. If \p contents + /// is NULL, go back to using the real the file system. + void (*_Nonnull set_file_contents)( + _Null_unspecified swiftide_api_connection_t connection, + const char *_Null_unspecified path, + const char *_Null_unspecified contents + ); + + /// Cancel the request with \p handle. + void (*_Nonnull cancel_request)( + _Null_unspecified swiftide_api_connection_t _conn, + _Null_unspecified swiftide_api_request_handle_t handle + ); + + _Null_unspecified swiftide_api_completion_request_t (*_Nonnull completion_request_create)( + const char *_Null_unspecified path, + uint32_t offset, + char *_Null_unspecified const *_Null_unspecified const compiler_args, + uint32_t num_compiler_args + ); + + void (*_Nonnull completion_request_dispose)( + _Null_unspecified swiftide_api_completion_request_t + ); + + void (*_Nonnull completion_request_set_annotate_result)( + _Null_unspecified swiftide_api_completion_request_t, + bool + ); + + void (*_Nonnull completion_request_set_include_objectliterals)( + _Null_unspecified swiftide_api_completion_request_t, + bool + ); + + void (*_Nonnull completion_request_set_add_inits_to_top_level)( + _Null_unspecified swiftide_api_completion_request_t, + bool + ); + + + void (*_Nonnull completion_request_set_add_call_with_no_default_args)( + _Null_unspecified swiftide_api_completion_request_t, + bool + ); + + /// Same as swiftide_complete but supports cancellation. + /// This request is identified by \p handle. Calling swiftide_cancel_request + /// with that handle cancels the request. + /// Note that the caller is responsible for creating a unique request handle. + /// This differs from the sourcekitd functions in which SourceKit creates a + /// unique handle and passes it to the client via an out parameter. + _Null_unspecified swiftide_api_completion_response_t (*_Nonnull complete_cancellable)( + _Null_unspecified swiftide_api_connection_t _conn, + _Null_unspecified swiftide_api_completion_request_t _req, + _Null_unspecified swiftide_api_request_handle_t handle + ); + + void (*_Nonnull completion_result_dispose)( + _Null_unspecified swiftide_api_completion_response_t + ); + + bool (*_Nonnull completion_result_is_error)( + _Null_unspecified swiftide_api_completion_response_t + ); + + /// Result has the same lifetime as the result. + const char *_Null_unspecified (*_Nonnull completion_result_get_error_description)( + _Null_unspecified swiftide_api_completion_response_t + ); + + bool (*_Nonnull completion_result_is_cancelled)( + _Null_unspecified swiftide_api_completion_response_t + ); + + /// Copies a string representation of the completion result. This string should + /// be disposed of with \c free when done. + const char *_Null_unspecified (*_Nonnull completion_result_description_copy)( + _Null_unspecified swiftide_api_completion_response_t + ); + + void (*_Nonnull completion_result_get_completions)( + _Null_unspecified swiftide_api_completion_response_t, + void (^_Null_unspecified completions_handler)( + const _Null_unspecified swiftide_api_completion_item_t *_Null_unspecified completions, + const char *_Null_unspecified *_Null_unspecified filter_names, + uint64_t num_completions + ) + ); + + _Null_unspecified swiftide_api_completion_item_t (*_Nonnull completion_result_get_completion_at_index)( + _Null_unspecified swiftide_api_completion_response_t, + uint64_t index + ); + + swiftide_api_completion_kind_t (*_Nonnull completion_result_get_kind)( + _Null_unspecified swiftide_api_completion_response_t + ); + + void (*_Nonnull completion_result_foreach_baseexpr_typename)( + _Null_unspecified swiftide_api_completion_response_t, + bool (^_Null_unspecified handler)(const char *_Null_unspecified ) + ); + + bool (*_Nonnull completion_result_is_reusing_astcontext)( + _Null_unspecified swiftide_api_completion_response_t + ); + + /// Copies a string representation of the completion item. This string should + /// be disposed of with \c free when done. + const char *_Null_unspecified (*_Nonnull completion_item_description_copy)( + _Null_unspecified swiftide_api_completion_item_t + ); + + + void (*_Nonnull completion_item_get_label)( + _Null_unspecified swiftide_api_completion_response_t, + _Null_unspecified swiftide_api_completion_item_t, + bool annotate, + void (^_Null_unspecified handler)(const char *_Null_unspecified) + ); + + void (*_Nonnull completion_item_get_source_text)( + _Null_unspecified swiftide_api_completion_response_t, + _Null_unspecified swiftide_api_completion_item_t, + void (^_Null_unspecified handler)(const char *_Null_unspecified) + ); + + void (*_Nonnull completion_item_get_type_name)( + _Null_unspecified swiftide_api_completion_response_t, + _Null_unspecified swiftide_api_completion_item_t, + bool annotate, + void (^_Null_unspecified handler)(const char *_Null_unspecified) + ); + + void (*_Nonnull completion_item_get_doc_brief)( + _Null_unspecified swiftide_api_completion_response_t, + _Null_unspecified swiftide_api_completion_item_t, + void (^_Null_unspecified handler)(const char *_Null_unspecified) + ); + + void (*_Nonnull completion_item_get_associated_usrs)( + _Null_unspecified swiftide_api_completion_response_t, + _Null_unspecified swiftide_api_completion_item_t, + void (^_Null_unspecified handler)(const char *_Null_unspecified *_Null_unspecified, uint64_t) + ); + + uint32_t (*_Nonnull completion_item_get_kind)( + _Null_unspecified swiftide_api_completion_item_t + ); + + uint32_t (*_Nonnull completion_item_get_associated_kind)( + _Null_unspecified swiftide_api_completion_item_t + ); + + uint32_t (*_Nonnull completion_item_get_semantic_context)( + _Null_unspecified swiftide_api_completion_item_t + ); + + uint32_t (*_Nonnull completion_item_get_flair)( + _Null_unspecified swiftide_api_completion_item_t + ); + + bool (*_Nonnull completion_item_is_not_recommended)( + _Null_unspecified swiftide_api_completion_item_t + ); + + uint32_t (*_Nonnull completion_item_not_recommended_reason)( + _Null_unspecified swiftide_api_completion_item_t + ); + + bool (*_Nonnull completion_item_has_diagnostic)( + _Null_unspecified swiftide_api_completion_item_t _item + ); + + void (*_Nonnull completion_item_get_diagnostic)( + _Null_unspecified swiftide_api_completion_response_t, + _Null_unspecified swiftide_api_completion_item_t, + void (^_Null_unspecified handler)(swiftide_api_completion_diagnostic_severity_t, const char *_Null_unspecified) + ); + + bool (*_Nonnull completion_item_is_system)( + _Null_unspecified swiftide_api_completion_item_t + ); + + void (*_Nonnull completion_item_get_module_name)( + _Null_unspecified swiftide_api_completion_response_t _response, + _Null_unspecified swiftide_api_completion_item_t _item, + void (^_Null_unspecified handler)(const char *_Null_unspecified) + ); + + uint32_t (*_Nonnull completion_item_get_num_bytes_to_erase)( + _Null_unspecified swiftide_api_completion_item_t + ); + + uint32_t (*_Nonnull completion_item_get_type_relation)( + _Null_unspecified swiftide_api_completion_item_t + ); + + /// Returns 0 for items not in an external module, and ~0u if the other module + /// is not imported or the depth is otherwise unknown. + uint32_t (*_Nonnull completion_item_import_depth)( + _Null_unspecified swiftide_api_completion_response_t, + _Null_unspecified swiftide_api_completion_item_t + ); + + _Null_unspecified swiftide_api_fuzzy_match_pattern_t (*_Nonnull fuzzy_match_pattern_create)( + const char *_Null_unspecified pattern + ); + + bool (*_Nonnull fuzzy_match_pattern_matches_candidate)( + _Null_unspecified swiftide_api_fuzzy_match_pattern_t pattern, + const char *_Null_unspecified candidate, + double *_Null_unspecified outScore + ); + + void (*_Nonnull fuzzy_match_pattern_dispose)( + _Null_unspecified swiftide_api_fuzzy_match_pattern_t + ); +} sourcekitd_ide_api_functions_t; + +#endif + diff --git a/Sources/Csourcekitd/include/module.modulemap b/Sources/Csourcekitd/include/module.modulemap index f842a3d80..d6c621c52 100644 --- a/Sources/Csourcekitd/include/module.modulemap +++ b/Sources/Csourcekitd/include/module.modulemap @@ -1,5 +1,6 @@ module Csourcekitd { - // header "sourcekitd.h" header "sourcekitd_functions.h" + header "CodeCompletionSwiftInterop.h" + header "plugin.h" export * } diff --git a/Sources/Csourcekitd/include/plugin.h b/Sources/Csourcekitd/include/plugin.h new file mode 100644 index 000000000..64b5607d2 --- /dev/null +++ b/Sources/Csourcekitd/include/plugin.h @@ -0,0 +1,473 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +#ifndef SWIFT_SOURCEKITD_PLUGIN_H +#define SWIFT_SOURCEKITD_PLUGIN_H + +#include "sourcekitd_functions.h" + + +typedef void *sourcekitd_api_variant_functions_t; + +typedef sourcekitd_api_variant_type_t (*sourcekitd_api_variant_functions_get_type_t)( + sourcekitd_api_variant_t obj +); +typedef bool (*sourcekitd_api_variant_functions_array_apply_t)( + sourcekitd_api_variant_t array, + _Null_unspecified sourcekitd_api_variant_array_applier_f_t applier, + void *_Null_unspecified context +); +typedef bool (*sourcekitd_api_variant_functions_array_get_bool_t)( + sourcekitd_api_variant_t array, + size_t index +); +typedef double (*sourcekitd_api_variant_functions_array_get_double_t)( + sourcekitd_api_variant_t array, + size_t index +); +typedef size_t (*sourcekitd_api_variant_functions_array_get_count_t)( + sourcekitd_api_variant_t array +); +typedef int64_t (*sourcekitd_api_variant_functions_array_get_int64_t)( + sourcekitd_api_variant_t array, + size_t index +); +typedef const char *_Null_unspecified (*sourcekitd_api_variant_functions_array_get_string_t)( + sourcekitd_api_variant_t array, + size_t index +); +typedef _Null_unspecified sourcekitd_api_uid_t (*sourcekitd_api_variant_functions_array_get_uid_t)( + sourcekitd_api_variant_t array, + size_t index +); +typedef sourcekitd_api_variant_t (*sourcekitd_api_variant_functions_array_get_value_t)( + sourcekitd_api_variant_t array, + size_t index +); +typedef bool (*sourcekitd_api_variant_functions_bool_get_value_t)( + sourcekitd_api_variant_t obj +); +typedef double (*sourcekitd_api_variant_functions_double_get_value_t)( + sourcekitd_api_variant_t obj +); +typedef bool (*sourcekitd_api_variant_functions_dictionary_apply_t)( + sourcekitd_api_variant_t dict, + _Null_unspecified sourcekitd_api_variant_dictionary_applier_f_t applier, + void *_Null_unspecified context +); +typedef bool (*sourcekitd_api_variant_functions_dictionary_get_bool_t)( + sourcekitd_api_variant_t dict, + _Null_unspecified sourcekitd_api_uid_t key +); +typedef double (*sourcekitd_api_variant_functions_dictionary_get_double_t)( + sourcekitd_api_variant_t dict, + _Null_unspecified sourcekitd_api_uid_t key +); +typedef int64_t (*sourcekitd_api_variant_functions_dictionary_get_int64_t)( + sourcekitd_api_variant_t dict, + _Null_unspecified sourcekitd_api_uid_t key +); +typedef const char *_Null_unspecified (*sourcekitd_api_variant_functions_dictionary_get_string_t)( + sourcekitd_api_variant_t dict, + _Null_unspecified sourcekitd_api_uid_t key +); +typedef sourcekitd_api_variant_t (*sourcekitd_api_variant_functions_dictionary_get_value_t)( + sourcekitd_api_variant_t dict, + _Null_unspecified sourcekitd_api_uid_t key +); +typedef _Null_unspecified sourcekitd_api_uid_t (*sourcekitd_api_variant_functions_dictionary_get_uid_t)( + sourcekitd_api_variant_t dict, + _Null_unspecified sourcekitd_api_uid_t key +); +typedef size_t (*sourcekitd_api_variant_functions_string_get_length_t)( + sourcekitd_api_variant_t obj +); +typedef const char *_Null_unspecified (*sourcekitd_api_variant_functions_string_get_ptr_t)( + sourcekitd_api_variant_t obj +); +typedef int64_t (*sourcekitd_api_variant_functions_int64_get_value_t)( + sourcekitd_api_variant_t obj +); +typedef _Null_unspecified sourcekitd_api_uid_t (*sourcekitd_api_variant_functions_uid_get_value_t)( + sourcekitd_api_variant_t obj +); +typedef size_t (*sourcekitd_api_variant_functions_data_get_size_t)( + sourcekitd_api_variant_t obj +); +typedef const void *_Null_unspecified (*sourcekitd_api_variant_functions_data_get_ptr_t)( + sourcekitd_api_variant_t obj +); + +/// Handle the request specified by the \c sourcekitd_api_object_t and keep track +/// of it using the \c sourcekitd_api_request_handle_t. If the cancellation handler +/// specified by \c sourcekitd_api_plugin_initialize_register_cancellation_handler +/// is called with the this request handle, the request should be cancelled. +typedef bool (^sourcekitd_api_cancellable_request_handler_t)( + _Null_unspecified sourcekitd_api_object_t, + _Null_unspecified sourcekitd_api_request_handle_t, + void (^_Null_unspecified __attribute__((swift_attr("@Sendable"))))(_Null_unspecified sourcekitd_api_response_t) +); +typedef void (^sourcekitd_api_cancellation_handler_t)(_Null_unspecified sourcekitd_api_request_handle_t); +typedef _Null_unspecified sourcekitd_api_uid_t (*sourcekitd_api_uid_get_from_cstr_t)(const char *_Null_unspecified string); +typedef const char *_Null_unspecified (*sourcekitd_api_uid_get_string_ptr_t)(_Null_unspecified sourcekitd_api_uid_t); + +typedef void *sourcekitd_api_plugin_initialize_params_t; +typedef void (*sourcekitd_api_plugin_initialize_t)( + _Null_unspecified sourcekitd_api_plugin_initialize_params_t +); + +typedef struct { + _Null_unspecified sourcekitd_api_variant_functions_t (*_Nonnull variant_functions_create)(void); + + void (*_Nonnull variant_functions_set_get_type)( + _Nonnull sourcekitd_api_variant_functions_t funcs, + _Nonnull sourcekitd_api_variant_functions_get_type_t f + ); + void (*_Nonnull variant_functions_set_array_apply)( + _Nonnull sourcekitd_api_variant_functions_t funcs, + _Nonnull sourcekitd_api_variant_functions_array_apply_t f + ); + void (*_Nonnull variant_functions_set_array_get_bool)( + _Nonnull sourcekitd_api_variant_functions_t funcs, + _Nonnull sourcekitd_api_variant_functions_array_get_bool_t f + ); + void (*_Nonnull variant_functions_set_array_get_double)( + _Nonnull sourcekitd_api_variant_functions_t funcs, + _Nonnull sourcekitd_api_variant_functions_array_get_double_t f + ); + void (*_Nonnull variant_functions_set_array_get_count)( + _Nonnull sourcekitd_api_variant_functions_t funcs, + _Nonnull sourcekitd_api_variant_functions_array_get_count_t f + ); + void (*_Nonnull variant_functions_set_array_get_int64)( + _Nonnull sourcekitd_api_variant_functions_t funcs, + _Nonnull sourcekitd_api_variant_functions_array_get_int64_t f + ); + void (*_Nonnull variant_functions_set_array_get_string)( + _Nonnull sourcekitd_api_variant_functions_t funcs, + _Nonnull sourcekitd_api_variant_functions_array_get_string_t f + ); + void (*_Nonnull variant_functions_set_array_get_uid)( + _Nonnull sourcekitd_api_variant_functions_t funcs, + _Nonnull sourcekitd_api_variant_functions_array_get_uid_t f + ); + void (*_Nonnull variant_functions_set_array_get_value)( + _Nonnull sourcekitd_api_variant_functions_t funcs, + _Nonnull sourcekitd_api_variant_functions_array_get_value_t f + ); + void (*_Nonnull variant_functions_set_bool_get_value)( + _Nonnull sourcekitd_api_variant_functions_t funcs, + _Nonnull sourcekitd_api_variant_functions_bool_get_value_t f + ); + void (*_Nonnull variant_functions_set_double_get_value)( + _Nonnull sourcekitd_api_variant_functions_t funcs, + _Nonnull sourcekitd_api_variant_functions_double_get_value_t f + ); + void (*_Nonnull variant_functions_set_dictionary_apply)( + _Nonnull sourcekitd_api_variant_functions_t funcs, + _Nonnull sourcekitd_api_variant_functions_dictionary_apply_t f + ); + void (*_Nonnull variant_functions_set_dictionary_get_bool)( + _Nonnull sourcekitd_api_variant_functions_t funcs, + _Nonnull sourcekitd_api_variant_functions_dictionary_get_bool_t f + ); + void (*_Nonnull variant_functions_set_dictionary_get_double)( + _Nonnull sourcekitd_api_variant_functions_t funcs, + _Nonnull sourcekitd_api_variant_functions_dictionary_get_double_t f + ); + void (*_Nonnull variant_functions_set_dictionary_get_int64)( + _Nonnull sourcekitd_api_variant_functions_t funcs, + _Nonnull sourcekitd_api_variant_functions_dictionary_get_int64_t f + ); + void (*_Nonnull variant_functions_set_dictionary_get_string)( + _Nonnull sourcekitd_api_variant_functions_t funcs, + _Nonnull sourcekitd_api_variant_functions_dictionary_get_string_t f + ); + void (*_Nonnull variant_functions_set_dictionary_get_value)( + _Nonnull sourcekitd_api_variant_functions_t funcs, + _Nonnull sourcekitd_api_variant_functions_dictionary_get_value_t f + ); + void (*_Nonnull variant_functions_set_dictionary_get_uid)( + _Nonnull sourcekitd_api_variant_functions_t funcs, + _Nonnull sourcekitd_api_variant_functions_dictionary_get_uid_t f + ); + void (*_Nonnull variant_functions_set_string_get_length)( + _Nonnull sourcekitd_api_variant_functions_t funcs, + _Nonnull sourcekitd_api_variant_functions_string_get_length_t f + ); + void (*_Nonnull variant_functions_set_string_get_ptr)( + _Nonnull sourcekitd_api_variant_functions_t funcs, + _Nonnull sourcekitd_api_variant_functions_string_get_ptr_t f + ); + void (*_Nonnull variant_functions_set_int64_get_value)( + _Nonnull sourcekitd_api_variant_functions_t funcs, + _Nonnull sourcekitd_api_variant_functions_int64_get_value_t f + ); + void (*_Nonnull variant_functions_set_uid_get_value)( + _Nonnull sourcekitd_api_variant_functions_t funcs, + _Nonnull sourcekitd_api_variant_functions_uid_get_value_t f + ); + void (*_Nonnull variant_functions_set_data_get_size)( + _Nonnull sourcekitd_api_variant_functions_t funcs, + _Nonnull sourcekitd_api_variant_functions_data_get_size_t f + ); + void (*_Nonnull variant_functions_set_data_get_ptr)( + _Nonnull sourcekitd_api_variant_functions_t funcs, + _Nonnull sourcekitd_api_variant_functions_data_get_ptr_t f + ); + + bool (*_Nonnull plugin_initialize_is_client_only)( + _Null_unspecified sourcekitd_api_plugin_initialize_params_t + ); + + uint64_t (*_Nonnull plugin_initialize_custom_buffer_start)( + _Null_unspecified sourcekitd_api_plugin_initialize_params_t + ); + + _Null_unspecified sourcekitd_api_uid_get_from_cstr_t (*_Nonnull plugin_initialize_uid_get_from_cstr)( + _Null_unspecified sourcekitd_api_plugin_initialize_params_t + ); + + _Null_unspecified sourcekitd_api_uid_get_string_ptr_t (*_Nonnull plugin_initialize_uid_get_string_ptr)( + _Null_unspecified sourcekitd_api_plugin_initialize_params_t + ); + + void (*_Nonnull plugin_initialize_register_custom_buffer)( + _Nonnull sourcekitd_api_plugin_initialize_params_t, + uint64_t kind, + _Nonnull sourcekitd_api_variant_functions_t funcs + ); +} sourcekitd_plugin_api_functions_t; + +typedef struct { + void (*_Nonnull plugin_initialize_register_cancellable_request_handler)( + _Nonnull sourcekitd_api_plugin_initialize_params_t, + _Nonnull sourcekitd_api_cancellable_request_handler_t + ); + + /// Adds a function that will be called when a request is cancelled. + /// The cancellation handler is called even for cancelled requests that are handled by + /// sourcekitd itself and not the plugin. If the plugin doesn't know the request + /// handle to be cancelled, it should ignore the cancellation request. + void (*_Nonnull plugin_initialize_register_cancellation_handler)( + _Nonnull sourcekitd_api_plugin_initialize_params_t, + _Nonnull sourcekitd_api_cancellation_handler_t + ); + + void *_Null_unspecified(*_Nonnull plugin_initialize_get_swift_ide_inspection_instance)( + _Null_unspecified sourcekitd_api_plugin_initialize_params_t + ); + + //============================================================================// + // Request + //============================================================================// + + sourcekitd_api_variant_type_t (*_Nonnull request_get_type)( + _Null_unspecified sourcekitd_api_object_t obj + ); + + _Null_unspecified sourcekitd_api_object_t (*_Nonnull request_dictionary_get_value)( + _Null_unspecified sourcekitd_api_object_t dict, + _Nonnull sourcekitd_api_uid_t key + ); + + /// The underlying C string for the specified key. NULL if the value for the + /// specified key is not a C string value or if there is no value for the + /// specified key. + const char *_Null_unspecified (*_Nonnull request_dictionary_get_string)( + _Nonnull sourcekitd_api_object_t dict, + _Nonnull sourcekitd_api_uid_t key + ); + + /// The underlying \c int64 value for the specified key. 0 if the + /// value for the specified key is not an integer value or if there is no + /// value for the specified key. + int64_t (*_Nonnull request_dictionary_get_int64)( + _Nonnull sourcekitd_api_object_t dict, + _Nonnull sourcekitd_api_uid_t key + ); + + /// The underlying \c bool value for the specified key. false if the + /// value for the specified key is not a Boolean value or if there is no + /// value for the specified key. + bool (*_Nonnull request_dictionary_get_bool)( + _Nonnull sourcekitd_api_object_t dict, + _Nonnull sourcekitd_api_uid_t key + ); + + _Null_unspecified sourcekitd_api_uid_t (*_Nonnull request_dictionary_get_uid)( + _Nonnull sourcekitd_api_object_t dict, + _Nonnull sourcekitd_api_uid_t key + ); + + size_t (*_Nonnull request_array_get_count)( + _Null_unspecified sourcekitd_api_object_t array + ); + + _Null_unspecified sourcekitd_api_object_t (*_Nonnull request_array_get_value)( + _Null_unspecified sourcekitd_api_object_t array, + size_t index + ); + + const char *_Null_unspecified (*_Nonnull request_array_get_string)( + _Null_unspecified sourcekitd_api_object_t array, + size_t index + ); + + int64_t (*_Nonnull request_array_get_int64)( + _Null_unspecified sourcekitd_api_object_t array, + size_t index + ); + + bool (*_Nonnull request_array_get_bool)( + _Null_unspecified sourcekitd_api_object_t array, + size_t index + ); + + _Null_unspecified sourcekitd_api_uid_t (*_Nonnull request_array_get_uid)( + _Null_unspecified sourcekitd_api_object_t array, + size_t index + ); + + int64_t (*_Nonnull request_int64_get_value)( + _Null_unspecified sourcekitd_api_object_t obj + ); + + bool (*_Nonnull request_bool_get_value)( + _Null_unspecified sourcekitd_api_object_t obj + ); + + size_t (*_Nonnull request_string_get_length)( + _Null_unspecified sourcekitd_api_object_t obj + ); + + const char *_Null_unspecified (*_Nonnull request_string_get_ptr)( + _Null_unspecified sourcekitd_api_object_t obj + ); + + _Null_unspecified sourcekitd_api_uid_t (*_Nonnull request_uid_get_value)( + _Null_unspecified sourcekitd_api_object_t obj + ); + + //============================================================================// + // Response + //============================================================================// + + _Nonnull sourcekitd_api_response_t (*_Nonnull response_retain)( + _Nonnull sourcekitd_api_response_t object + ); + + _Null_unspecified sourcekitd_api_response_t (*_Nonnull response_error_create)( + sourcekitd_api_error_t kind, + const char *_Null_unspecified description + ); + + _Nonnull sourcekitd_api_response_t (*_Nonnull response_dictionary_create)( + const _Null_unspecified sourcekitd_api_uid_t *_Null_unspecified keys, + const _Null_unspecified sourcekitd_api_response_t *_Null_unspecified values, + size_t count + ); + + void (*_Nonnull response_dictionary_set_value)( + _Nonnull sourcekitd_api_response_t dict, + _Nonnull sourcekitd_api_uid_t key, + _Nonnull sourcekitd_api_response_t value + ); + + void (*_Nonnull response_dictionary_set_string)( + _Nonnull sourcekitd_api_response_t dict, + _Nonnull sourcekitd_api_uid_t key, + const char *_Nonnull string + ); + + void (*_Nonnull response_dictionary_set_stringbuf)( + _Nonnull sourcekitd_api_response_t dict, + _Nonnull sourcekitd_api_uid_t key, + const char *_Nonnull buf, + size_t length + ); + + void (*_Nonnull response_dictionary_set_int64)( + _Nonnull sourcekitd_api_response_t dict, + _Nonnull sourcekitd_api_uid_t key, + int64_t val + ); + + void (*_Nonnull response_dictionary_set_bool)( + _Nonnull sourcekitd_api_response_t dict, + _Nonnull sourcekitd_api_uid_t key, + bool val + ); + + void (*_Nonnull response_dictionary_set_double)( + _Nonnull sourcekitd_api_response_t dict, + _Nonnull sourcekitd_api_uid_t key, + double val + ); + + void (*_Nonnull response_dictionary_set_uid)( + _Nonnull sourcekitd_api_response_t dict, + _Nonnull sourcekitd_api_uid_t key, + _Nonnull sourcekitd_api_uid_t uid + ); + + _Nonnull sourcekitd_api_response_t (*_Nonnull response_array_create)( + const _Null_unspecified sourcekitd_api_response_t *_Null_unspecified objects, + size_t count + ); + + void (*_Nonnull response_array_set_value)( + _Nonnull sourcekitd_api_response_t array, + size_t index, + _Nonnull sourcekitd_api_response_t value + ); + + void (*_Nonnull response_array_set_string)( + _Nonnull sourcekitd_api_response_t array, + size_t index, + const char *_Nonnull string + ); + + void (*_Nonnull response_array_set_stringbuf)( + _Nonnull sourcekitd_api_response_t array, + size_t index, + const char *_Nonnull buf, + size_t length + ); + + void (*_Nonnull response_array_set_int64)( + _Nonnull sourcekitd_api_response_t array, + size_t index, + int64_t val + ); + + void (*_Nonnull response_array_set_double)( + _Nonnull sourcekitd_api_response_t array, + size_t index, + double val + ); + + void (*_Nonnull response_array_set_uid)( + _Nonnull sourcekitd_api_response_t array, + size_t index, + _Nonnull sourcekitd_api_uid_t uid + ); + + void (*_Nonnull response_dictionary_set_custom_buffer)( + _Nonnull sourcekitd_api_response_t dict, + _Nonnull sourcekitd_api_uid_t key, + const void *_Nonnull ptr, + size_t size + ); +} sourcekitd_service_plugin_api_functions_t; + +#endif diff --git a/Sources/Csourcekitd/include/sourcekitd_functions.h b/Sources/Csourcekitd/include/sourcekitd_functions.h index bddc58e7e..ec597e6f2 100644 --- a/Sources/Csourcekitd/include/sourcekitd_functions.h +++ b/Sources/Csourcekitd/include/sourcekitd_functions.h @@ -38,11 +38,43 @@ typedef enum { SOURCEKITD_API_VARIANT_TYPE_STRING = 4, SOURCEKITD_API_VARIANT_TYPE_UID = 5, SOURCEKITD_API_VARIANT_TYPE_BOOL = 6, - // Reserved for future addition - // SOURCEKITD_VARIANT_TYPE_DOUBLE = 7, + SOURCEKITD_API_VARIANT_TYPE_DOUBLE = 7, SOURCEKITD_API_VARIANT_TYPE_DATA = 8, } sourcekitd_api_variant_type_t; +typedef void *sourcekitd_api_variant_functions_t; + +typedef sourcekitd_api_variant_type_t (*sourcekitd_api_variant_functions_get_type_t)( + sourcekitd_api_variant_t obj +); + +typedef size_t (*sourcekitd_api_variant_functions_array_get_count_t)( + sourcekitd_api_variant_t array +); + +typedef sourcekitd_api_variant_t (*sourcekitd_api_variant_functions_array_get_value_t)( + sourcekitd_api_variant_t array, + size_t index +); + +typedef bool (*sourcekitd_api_variant_array_applier_f_t)( + size_t index, + sourcekitd_api_variant_t value, + void *_Null_unspecified context +); + +typedef bool (*sourcekitd_api_variant_dictionary_applier_f_t)( + _Null_unspecified sourcekitd_api_uid_t key, + sourcekitd_api_variant_t value, + void *_Null_unspecified context +); + +typedef bool (*sourcekitd_api_variant_functions_dictionary_apply_t)( + sourcekitd_api_variant_t dict, + _Null_unspecified sourcekitd_api_variant_dictionary_applier_f_t applier, + void *_Null_unspecified context +); + typedef enum { SOURCEKITD_API_ERROR_CONNECTION_INTERRUPTED = 1, SOURCEKITD_API_ERROR_REQUEST_INVALID = 2, @@ -62,9 +94,17 @@ typedef const char *_Nullable (^sourcekitd_api_str_from_uid_handler_t)( sourcekitd_api_uid_t _Nullable uid ); +typedef void *sourcekitd_api_plugin_initialize_params_t; + typedef struct { void (*_Nonnull initialize)(void); void (*_Nonnull shutdown)(void); + + void (*_Nullable register_plugin_path)( + const char *_Nullable clientPlugin, + const char *_Nullable servicePlugin + ); + _Null_unspecified sourcekitd_api_uid_t (*_Nonnull uid_get_from_cstr)( const char *_Nonnull string ); @@ -231,6 +271,9 @@ typedef struct { bool (*_Nonnull variant_bool_get_value)( sourcekitd_api_variant_t obj ); + double (*_Nullable variant_double_get_value)( + sourcekitd_api_variant_t obj + ); size_t (*_Nonnull variant_string_get_length)( sourcekitd_api_variant_t obj ); diff --git a/Sources/SKLogging/CustomLogStringConvertible.swift b/Sources/SKLogging/CustomLogStringConvertible.swift index 00127dddb..71197c626 100644 --- a/Sources/SKLogging/CustomLogStringConvertible.swift +++ b/Sources/SKLogging/CustomLogStringConvertible.swift @@ -11,13 +11,15 @@ //===----------------------------------------------------------------------===// #if compiler(>=6) -import Crypto package import Foundation #else -import Crypto import Foundation #endif +#if !NO_CRYPTO_DEPENDENCY +import Crypto +#endif + /// An object that can printed for logging and also offers a redacted description /// when logging in contexts in which private information shouldn't be captured. package protocol CustomLogStringConvertible: CustomStringConvertible, Sendable { @@ -69,8 +71,12 @@ extension String { /// A hash value that can be logged in a redacted description without /// disclosing any private information about the string. package var hashForLogging: String { + #if NO_CRYPTO_DEPENDENCY + return "" + #else let hash = SHA256.hash(data: Data(self.utf8)).prefix(8).map { String(format: "%02x", $0) }.joined() return "" + #endif } } diff --git a/Sources/SKLoggingForPlugin b/Sources/SKLoggingForPlugin new file mode 120000 index 000000000..3928b983a --- /dev/null +++ b/Sources/SKLoggingForPlugin @@ -0,0 +1 @@ +SKLogging/ \ No newline at end of file diff --git a/Sources/SKTestSupport/Assertions.swift b/Sources/SKTestSupport/Assertions.swift index 94533e1cc..bf69ca2af 100644 --- a/Sources/SKTestSupport/Assertions.swift +++ b/Sources/SKTestSupport/Assertions.swift @@ -61,6 +61,24 @@ package func assertThrowsError( } } +/// Asserts that executing `expression` throws an error and that the error's string representation matches `expectedMessage`. +public func assertThrowsError( + _ expression: @autoclosure () async throws -> T, + expectedMessage: Regex, + file: StaticString = #filePath, + line: UInt = #line +) async { + await assertThrowsError(try await expression(), file: file, line: line) { error in + let errorString = String(reflecting: error) + XCTAssert( + try! expectedMessage.firstMatch(in: errorString) != nil, + "Expected error to contain '\(expectedMessage)' but received '\(errorString)'", + file: file, + line: line + ) + } +} + /// Same as `XCTAssertEqual` but doesn't take autoclosures and thus `expression1` /// and `expression2` can contain `await`. package func assertEqual( diff --git a/Sources/SKTestSupport/MultiFileTestProject.swift b/Sources/SKTestSupport/MultiFileTestProject.swift index 7f6b0b5dc..2f5ecb235 100644 --- a/Sources/SKTestSupport/MultiFileTestProject.swift +++ b/Sources/SKTestSupport/MultiFileTestProject.swift @@ -58,12 +58,12 @@ package struct RelativeFileLocation: Hashable, ExpressibleByStringLiteral { /// The temporary files will be deleted when the `TestSourceKitLSPClient` is destructed. package class MultiFileTestProject { /// Information necessary to open a file in the LSP server by its filename. - private struct FileData { + package struct FileData { /// The URI at which the file is stored on disk. - let uri: DocumentURI + package let uri: DocumentURI /// The contents of the file including location markers. - let markedText: String + package let markedText: String } package let testClient: TestSourceKitLSPClient @@ -79,29 +79,10 @@ package class MultiFileTestProject { /// The directory in which the temporary files are being placed. package let scratchDirectory: URL - /// Writes the specified files to a temporary directory on disk and creates a `TestSourceKitLSPClient` for that - /// temporary directory. - /// - /// The file contents can contain location markers, which are returned when opening a document using - /// ``openDocument(_:)``. - /// - /// File contents can also contain `$TEST_DIR`, which gets replaced by the temporary directory. - package init( + package static func writeFilesToDisk( files: [RelativeFileLocation: String], - workspaces: (_ scratchDirectory: URL) async throws -> [WorkspaceFolder] = { - [WorkspaceFolder(uri: DocumentURI($0))] - }, - initializationOptions: LSPAny? = nil, - capabilities: ClientCapabilities = ClientCapabilities(), - options: SourceKitLSPOptions = .testDefault(), - testHooks: TestHooks = TestHooks(), - enableBackgroundIndexing: Bool = false, - usePullDiagnostics: Bool = true, - preInitialization: ((TestSourceKitLSPClient) -> Void)? = nil, - cleanUp: (@Sendable () -> Void)? = nil, - testName: String = #function - ) async throws { - scratchDirectory = try testScratchDir(testName: testName) + scratchDirectory: URL + ) throws -> [String: FileData] { try FileManager.default.createDirectory(at: scratchDirectory, withIntermediateDirectories: true) var fileData: [String: FileData] = [:] @@ -134,7 +115,33 @@ package class MultiFileTestProject { ) } } - self.fileData = fileData + return fileData + } + + /// Writes the specified files to a temporary directory on disk and creates a `TestSourceKitLSPClient` for that + /// temporary directory. + /// + /// The file contents can contain location markers, which are returned when opening a document using + /// ``openDocument(_:)``. + /// + /// File contents can also contain `$TEST_DIR`, which gets replaced by the temporary directory. + package init( + files: [RelativeFileLocation: String], + workspaces: (_ scratchDirectory: URL) async throws -> [WorkspaceFolder] = { + [WorkspaceFolder(uri: DocumentURI($0))] + }, + initializationOptions: LSPAny? = nil, + capabilities: ClientCapabilities = ClientCapabilities(), + options: SourceKitLSPOptions = .testDefault(), + testHooks: TestHooks = TestHooks(), + enableBackgroundIndexing: Bool = false, + usePullDiagnostics: Bool = true, + preInitialization: ((TestSourceKitLSPClient) -> Void)? = nil, + cleanUp: (@Sendable () -> Void)? = nil, + testName: String = #function + ) async throws { + scratchDirectory = try testScratchDir(testName: testName) + self.fileData = try Self.writeFilesToDisk(files: files, scratchDirectory: scratchDirectory) self.testClient = try await TestSourceKitLSPClient( options: options, diff --git a/Sources/SKTestSupport/TestSourceKitLSPClient.swift b/Sources/SKTestSupport/TestSourceKitLSPClient.swift index b902705a8..b6cb8a3f8 100644 --- a/Sources/SKTestSupport/TestSourceKitLSPClient.swift +++ b/Sources/SKTestSupport/TestSourceKitLSPClient.swift @@ -39,8 +39,12 @@ import XCTest #endif extension SourceKitLSPOptions { - package static func testDefault(experimentalFeatures: Set? = nil) -> SourceKitLSPOptions { + package static func testDefault( + backgroundIndexing: Bool = true, + experimentalFeatures: Set? = nil + ) -> SourceKitLSPOptions { return SourceKitLSPOptions( + backgroundIndexing: backgroundIndexing, experimentalFeatures: experimentalFeatures, swiftPublishDiagnosticsDebounceDuration: 0, workDoneProgressDebounceDuration: 0 @@ -426,7 +430,7 @@ package final class TestSourceKitLSPClient: MessageHandler, Sendable { package struct DocumentPositions { private let positions: [String: Position] - fileprivate init(markers: [String: Int], textWithoutMarkers: String) { + package init(markers: [String: Int], textWithoutMarkers: String) { if markers.isEmpty { // No need to build a line table if we don't have any markers. positions = [:] diff --git a/Sources/SKTestSupport/Utils.swift b/Sources/SKTestSupport/Utils.swift index 48dc1ff63..13530ddec 100644 --- a/Sources/SKTestSupport/Utils.swift +++ b/Sources/SKTestSupport/Utils.swift @@ -64,25 +64,35 @@ extension DocumentURI { package let cleanScratchDirectories = (ProcessInfo.processInfo.environment["SOURCEKIT_LSP_KEEP_TEST_SCRATCH_DIR"] == nil) -/// An empty directory in which a test with `#function` name `testName` can store temporary data. -package func testScratchDir(testName: String = #function) throws -> URL { - let testBaseName = testName.prefix(while: \.isLetter) - +package func testScratchName(testName: String = #function) -> String { var uuid = UUID().uuidString[...] if let firstDash = uuid.firstIndex(of: "-") { uuid = uuid[.. URL { + #if os(Windows) + // Use a shorter test scratch dir name on Windows to not exceed MAX_PATH length + let testScratchDirsName = "lsp-test" + #else + let testScratchDirsName = "sourcekit-lsp-test-scratch" #endif + + let url = try FileManager.default.temporaryDirectory.realpath + .appendingPathComponent(testScratchDirsName) + .appendingPathComponent(testScratchName(testName: testName), isDirectory: true) + try? FileManager.default.removeItem(at: url) try FileManager.default.createDirectory(at: url, withIntermediateDirectories: true) return url diff --git a/Sources/SKUtilities/LineTable.swift b/Sources/SKUtilities/LineTable.swift index 67296f993..2d9abd05d 100644 --- a/Sources/SKUtilities/LineTable.swift +++ b/Sources/SKUtilities/LineTable.swift @@ -114,6 +114,52 @@ extension LineTable { let (toLine, toOff) = lineAndUTF16ColumnOf(end, fromLine: fromLine) self.replace(fromLine: fromLine, utf16Offset: fromOff, toLine: toLine, utf16Offset: toOff, with: replacement) } + + /// Replace the line table's `content` in the given range and update the line data. + /// + /// - parameter fromLine: Starting line number (zero-based). + /// - parameter fromOff: Starting UTF-8 column offset (zero-based). + /// - parameter toLine: Ending line number (zero-based). + /// - parameter toOff: Ending UTF-8 column offset (zero-based). + /// - parameter replacement: The new text for the given range. + @inlinable + mutating package func replace( + fromLine: Int, + utf8Offset fromOff: Int, + toLine: Int, + utf8Offset toOff: Int, + with replacement: String + ) { + let start = content.utf8.index(impl[fromLine], offsetBy: fromOff) + let end = content.utf8.index(impl[toLine], offsetBy: toOff) + + var newText = self.content + newText.replaceSubrange(start..(compute: () -> WrappedValue) -> WrappedValue where T == Optional { + return withLock { value in + if let value { + return value + } + let computed = compute() + value = computed + return computed + } + } +} + /// Wrapper for sourcekitd, taking care of initialization, shutdown, and notification handler /// multiplexing. /// @@ -41,46 +56,105 @@ package actor DynamicallyLoadedSourceKitD: SourceKitD { /// The sourcekitd API functions. package let api: sourcekitd_api_functions_t - /// Convenience for accessing known keys. - package let keys: sourcekitd_api_keys + private let pluginApiResult: Result + package nonisolated var pluginApi: sourcekitd_plugin_api_functions_t { try! pluginApiResult.get() } - /// Convenience for accessing known keys. - package let requests: sourcekitd_api_requests + private let servicePluginApiResult: Result + package nonisolated var servicePluginApi: sourcekitd_service_plugin_api_functions_t { + try! servicePluginApiResult.get() + } + + private let ideApiResult: Result + package nonisolated var ideApi: sourcekitd_ide_api_functions_t { try! ideApiResult.get() } /// Convenience for accessing known keys. - package let values: sourcekitd_api_values + /// + /// These need to be computed dynamically so that a client has the chance to register a UID handler between the + /// initialization of the SourceKit plugin and the first request being handled by it. + private let _keys: ThreadSafeBox = ThreadSafeBox(initialValue: nil) + package nonisolated var keys: sourcekitd_api_keys { + _keys.computeIfNil { sourcekitd_api_keys(api: self.api) } + } + + /// Convenience for accessing known request names. + /// + /// These need to be computed dynamically so that a client has the chance to register a UID handler between the + /// initialization of the SourceKit plugin and the first request being handled by it. + private let _requests: ThreadSafeBox = ThreadSafeBox(initialValue: nil) + package nonisolated var requests: sourcekitd_api_requests { + _requests.computeIfNil { sourcekitd_api_requests(api: self.api) } + } + + /// Convenience for accessing known request/response values. + /// + /// These need to be computed dynamically so that a client has the chance to register a UID handler between the + /// initialization of the SourceKit plugin and the first request being handled by it. + private let _values: ThreadSafeBox = ThreadSafeBox(initialValue: nil) + package nonisolated var values: sourcekitd_api_values { + _values.computeIfNil { sourcekitd_api_values(api: self.api) } + } private nonisolated let notificationHandlingQueue = AsyncQueue() /// List of notification handlers that will be called for each notification. private var notificationHandlers: [WeakSKDNotificationHandler] = [] - package static func getOrCreate(dylibPath: URL) async throws -> SourceKitD { + package static func getOrCreate( + dylibPath: URL, + pluginPaths: PluginPaths? = nil + ) async throws -> SourceKitD { try await SourceKitDRegistry.shared - .getOrAdd(dylibPath, create: { try DynamicallyLoadedSourceKitD(dylib: dylibPath) }) + .getOrAdd( + dylibPath, + pluginPaths: pluginPaths, + create: { try DynamicallyLoadedSourceKitD(dylib: dylibPath, pluginPaths: pluginPaths) } + ) } - init(dylib path: URL) throws { - self.path = path + private init(dylib path: URL, pluginPaths: PluginPaths?) throws { #if os(Windows) - self.dylib = try dlopen(path.filePath, mode: []) + let dlopenModes: DLOpenFlags = [] #else - self.dylib = try dlopen(path.filePath, mode: [.lazy, .local, .first]) + let dlopenModes: DLOpenFlags = [.lazy, .local, .first] #endif - self.api = try sourcekitd_api_functions_t(self.dylib) - self.keys = sourcekitd_api_keys(api: self.api) - self.requests = sourcekitd_api_requests(api: self.api) - self.values = sourcekitd_api_values(api: self.api) - - self.api.initialize() - self.api.set_notification_handler { [weak self] rawResponse in - guard let self, let rawResponse else { return } - let response = SKDResponse(rawResponse, sourcekitd: self) - self.notificationHandlingQueue.async { - let handlers = await self.notificationHandlers.compactMap(\.value) - - for handler in handlers { - handler.notification(response) + let dlhandle = try dlopen(path.filePath, mode: []) + try self.init( + dlhandle: dlhandle, + path: path, + pluginPaths: pluginPaths, + initialize: true + ) + } + + package init(dlhandle: DLHandle, path: URL, pluginPaths: PluginPaths?, initialize: Bool) throws { + self.path = path + self.dylib = dlhandle + let api = try sourcekitd_api_functions_t(dlhandle) + self.api = api + + // We load the plugin-related functions eagerly so the members are initialized and we don't have data races on first + // access to eg. `pluginApi`. But if one of the functions is missing, we will only emit that error when that family + // of functions is being used. For example, it is expected that the plugin functions are not available in + // SourceKit-LSP. + self.ideApiResult = Result(catching: { try sourcekitd_ide_api_functions_t(dlhandle) }) + self.pluginApiResult = Result(catching: { try sourcekitd_plugin_api_functions_t(dlhandle) }) + self.servicePluginApiResult = Result(catching: { try sourcekitd_service_plugin_api_functions_t(dlhandle) }) + + api.register_plugin_path?(pluginPaths?.clientPlugin.path, pluginPaths?.servicePlugin.path) + if initialize { + self.api.initialize() + } + + if initialize { + self.api.set_notification_handler { [weak self] rawResponse in + guard let self, let rawResponse else { return } + let response = SKDResponse(rawResponse, sourcekitd: self) + self.notificationHandlingQueue.async { + let handlers = await self.notificationHandlers.compactMap(\.value) + + for handler in handlers { + handler.notification(response) + } } } } diff --git a/Sources/SourceKitD/SKDResponseArray.swift b/Sources/SourceKitD/SKDResponseArray.swift index 1a050c8e5..fe51a2905 100644 --- a/Sources/SourceKitD/SKDResponseArray.swift +++ b/Sources/SourceKitD/SKDResponseArray.swift @@ -91,6 +91,16 @@ package final class SKDResponseArray: Sendable { } return nil } + + public var asStringArray: [String] { + var result: [String] = [] + for i in 0.. String? { return sourcekitd.api.variant_dictionary_get_string(dict, key).map(String.init(cString:)) } + package subscript(key: sourcekitd_api_uid_t) -> Int? { let value = sourcekitd.api.variant_dictionary_get_value(dict, key) if sourcekitd.api.variant_get_type(value) == SOURCEKITD_API_VARIANT_TYPE_INT64 { @@ -50,6 +51,7 @@ package final class SKDResponseDictionary: Sendable { return nil } } + package subscript(key: sourcekitd_api_uid_t) -> Bool? { let value = sourcekitd.api.variant_dictionary_get_value(dict, key) if sourcekitd.api.variant_get_type(value) == SOURCEKITD_API_VARIANT_TYPE_BOOL { @@ -58,9 +60,20 @@ package final class SKDResponseDictionary: Sendable { return nil } } + + public subscript(key: sourcekitd_api_uid_t) -> Double? { + let value = sourcekitd.api.variant_dictionary_get_value(dict, key) + if sourcekitd.api.variant_get_type(value) == SOURCEKITD_API_VARIANT_TYPE_DOUBLE { + return sourcekitd.api.variant_double_get_value!(value) + } else { + return nil + } + } + package subscript(key: sourcekitd_api_uid_t) -> sourcekitd_api_uid_t? { return sourcekitd.api.variant_dictionary_get_uid(dict, key) } + package subscript(key: sourcekitd_api_uid_t) -> SKDResponseArray? { let value = sourcekitd.api.variant_dictionary_get_value(dict, key) if sourcekitd.api.variant_get_type(value) == SOURCEKITD_API_VARIANT_TYPE_ARRAY { diff --git a/Sources/SourceKitD/SourceKitD.swift b/Sources/SourceKitD/SourceKitD.swift index 1e3e1530b..8a51b76ee 100644 --- a/Sources/SourceKitD/SourceKitD.swift +++ b/Sources/SourceKitD/SourceKitD.swift @@ -10,10 +10,12 @@ // //===----------------------------------------------------------------------===// +import SKLogging + #if compiler(>=6) package import Csourcekitd import Dispatch -import Foundation +package import Foundation import SwiftExtensions #else import Csourcekitd @@ -27,6 +29,24 @@ fileprivate struct SourceKitDRequestHandle: Sendable { nonisolated(unsafe) let handle: sourcekitd_api_request_handle_t } +package struct PluginPaths: Equatable, CustomLogStringConvertible { + let clientPlugin: URL + let servicePlugin: URL + + package init(clientPlugin: URL, servicePlugin: URL) { + self.clientPlugin = clientPlugin + self.servicePlugin = servicePlugin + } + + package var description: String { + "(client: \(clientPlugin), service: \(servicePlugin))" + } + + var redactedDescription: String { + "(client: \(clientPlugin.description.hashForLogging), service: \(servicePlugin.description.hashForLogging))" + } +} + /// Access to sourcekitd API, taking care of initialization, shutdown, and notification handler /// multiplexing. /// @@ -39,6 +59,23 @@ package protocol SourceKitD: AnyObject, Sendable { /// The sourcekitd API functions. var api: sourcekitd_api_functions_t { get } + /// General API for the SourceKit service and client framework, eg. for plugin initialization and to set up custom + /// variant functions. + /// + /// This must not be referenced outside of `SwiftSourceKitPlugin`, `SwiftSourceKitPluginCommon`, or + /// `SwiftSourceKitClientPlugin`. + var pluginApi: sourcekitd_plugin_api_functions_t { get } + + /// The API with which the SourceKit plugin handles requests. + /// + /// This must not be referenced outside of `SwiftSourceKitPlugin`. + var servicePluginApi: sourcekitd_service_plugin_api_functions_t { get } + + /// The API with which the SourceKit plugin communicates with the type-checker in-process. + /// + /// This must not be referenced outside of `SwiftSourceKitPlugin`. + var ideApi: sourcekitd_ide_api_functions_t { get } + /// Convenience for accessing known keys. var keys: sourcekitd_api_keys { get } diff --git a/Sources/SourceKitD/SourceKitDRegistry.swift b/Sources/SourceKitD/SourceKitDRegistry.swift index 3b28ace27..e4d9c003c 100644 --- a/Sources/SourceKitD/SourceKitDRegistry.swift +++ b/Sources/SourceKitD/SourceKitDRegistry.swift @@ -10,6 +10,8 @@ // //===----------------------------------------------------------------------===// +import SKLogging + #if compiler(>=6) package import Foundation #else @@ -28,10 +30,10 @@ import Foundation package actor SourceKitDRegistry { /// Mapping from path to active SourceKitD instance. - private var active: [URL: SourceKitD] = [:] + private var active: [URL: (pluginPaths: PluginPaths?, sourcekitd: SourceKitD)] = [:] /// Instances that have been unregistered, but may be resurrected if accessed before destruction. - private var cemetary: [URL: WeakSourceKitD] = [:] + private var cemetary: [URL: (pluginPaths: PluginPaths?, sourcekitd: WeakSourceKitD)] = [:] /// Initialize an empty registry. package init() {} @@ -42,18 +44,29 @@ package actor SourceKitDRegistry { /// Returns the existing SourceKitD for the given path, or creates it and registers it. package func getOrAdd( _ key: URL, - create: @Sendable () throws -> SourceKitD - ) rethrows -> SourceKitD { + pluginPaths: PluginPaths?, + create: () throws -> SourceKitD + ) async rethrows -> SourceKitD { if let existing = active[key] { - return existing + if existing.pluginPaths != pluginPaths { + logger.fault( + "Already created SourceKitD with plugin paths \(existing.pluginPaths?.forLogging), now requesting incompatible plugin paths \(pluginPaths.forLogging)" + ) + } + return existing.sourcekitd } - if let resurrected = cemetary[key]?.value { + if let resurrected = cemetary[key], let resurrectedSourcekitD = resurrected.sourcekitd.value { cemetary[key] = nil - active[key] = resurrected - return resurrected + if resurrected.pluginPaths != pluginPaths { + logger.fault( + "Already created SourceKitD with plugin paths \(resurrected.pluginPaths?.forLogging), now requesting incompatible plugin paths \(pluginPaths.forLogging)" + ) + } + active[key] = (resurrected.pluginPaths, resurrectedSourcekitD) + return resurrectedSourcekitD } let newValue = try create() - active[key] = newValue + active[key] = (pluginPaths, newValue) return newValue } @@ -67,10 +80,10 @@ package actor SourceKitDRegistry { package func remove(_ key: URL) -> SourceKitD? { let existing = active.removeValue(forKey: key) if let existing = existing { - assert(self.cemetary[key]?.value == nil) - cemetary[key] = WeakSourceKitD(value: existing) + assert(self.cemetary[key]?.sourcekitd.value == nil) + cemetary[key] = (existing.pluginPaths, WeakSourceKitD(value: existing.sourcekitd)) } - return existing + return existing?.sourcekitd } } diff --git a/Sources/SourceKitD/dlopen.swift b/Sources/SourceKitD/dlopen.swift index 3c402c487..fe595e59a 100644 --- a/Sources/SourceKitD/dlopen.swift +++ b/Sources/SourceKitD/dlopen.swift @@ -34,6 +34,10 @@ package final class DLHandle: Sendable { #endif } + #if !os(Windows) + package static let rtldDefault = DLHandle(rawValue: Handle(handle: UnsafeMutableRawPointer(bitPattern: -2)!)) + #endif + fileprivate let rawValue: ThreadSafeBox fileprivate init(rawValue: Handle) { diff --git a/Sources/SourceKitD/sourcekitd_functions.swift b/Sources/SourceKitD/sourcekitd_functions.swift index 99382a7e2..6edb883c2 100644 --- a/Sources/SourceKitD/sourcekitd_functions.swift +++ b/Sources/SourceKitD/sourcekitd_functions.swift @@ -28,6 +28,7 @@ extension sourcekitd_api_functions_t { self.init( initialize: try loadRequired("sourcekitd_initialize"), shutdown: try loadRequired("sourcekitd_shutdown"), + register_plugin_path: loadOptional("sourcekitd_register_plugin_path"), uid_get_from_cstr: try loadRequired("sourcekitd_uid_get_from_cstr"), uid_get_from_buf: try loadRequired("sourcekitd_uid_get_from_buf"), uid_get_length: try loadRequired("sourcekitd_uid_get_length"), @@ -71,6 +72,7 @@ extension sourcekitd_api_functions_t { variant_array_get_uid: try loadRequired("sourcekitd_variant_array_get_uid"), variant_int64_get_value: try loadRequired("sourcekitd_variant_int64_get_value"), variant_bool_get_value: try loadRequired("sourcekitd_variant_bool_get_value"), + variant_double_get_value: loadOptional("sourcekitd_variant_double_get_value"), variant_string_get_length: try loadRequired("sourcekitd_variant_string_get_length"), variant_string_get_ptr: try loadRequired("sourcekitd_variant_string_get_ptr"), variant_data_get_size: loadOptional("sourcekitd_variant_data_get_size"), @@ -91,3 +93,194 @@ extension sourcekitd_api_functions_t { } } + +extension sourcekitd_ide_api_functions_t { + package init(_ sourcekitd: DLHandle) throws { + func loadRequired(_ symbol: String) throws -> T { + guard let sym: T = dlsym(sourcekitd, symbol: symbol) else { + throw SKDError.missingRequiredSymbol(symbol) + } + return sym + } + + func loadOptional(_ symbol: String) -> T? { + return dlsym(sourcekitd, symbol: symbol) + } + self.init( + connection_create_with_inspection_instance: try loadRequired( + "swiftide_connection_create_with_inspection_instance" + ), + connection_dispose: try loadRequired("swiftide_connection_dispose"), + connection_mark_cached_compiler_instance_should_be_invalidated: try loadRequired( + "swiftide_connection_mark_cached_compiler_instance_should_be_invalidated" + ), + set_file_contents: try loadRequired("swiftide_set_file_contents"), + cancel_request: try loadRequired("swiftide_cancel_request"), + completion_request_create: try loadRequired("swiftide_completion_request_create"), + completion_request_dispose: try loadRequired("swiftide_completion_request_dispose"), + completion_request_set_annotate_result: try loadRequired("swiftide_completion_request_set_annotate_result"), + completion_request_set_include_objectliterals: try loadRequired( + "swiftide_completion_request_set_include_objectliterals" + ), + completion_request_set_add_inits_to_top_level: try loadRequired( + "swiftide_completion_request_set_add_inits_to_top_level" + ), + completion_request_set_add_call_with_no_default_args: try loadRequired( + "swiftide_completion_request_set_add_call_with_no_default_args" + ), + complete_cancellable: try loadRequired("swiftide_complete_cancellable"), + completion_result_dispose: try loadRequired("swiftide_completion_result_dispose"), + completion_result_is_error: try loadRequired("swiftide_completion_result_is_error"), + completion_result_get_error_description: try loadRequired("swiftide_completion_result_get_error_description"), + completion_result_is_cancelled: try loadRequired("swiftide_completion_result_is_cancelled"), + completion_result_description_copy: try loadRequired("swiftide_completion_result_description_copy"), + completion_result_get_completions: try loadRequired("swiftide_completion_result_get_completions"), + completion_result_get_completion_at_index: try loadRequired("swiftide_completion_result_get_completion_at_index"), + completion_result_get_kind: try loadRequired("swiftide_completion_result_get_kind"), + completion_result_foreach_baseexpr_typename: try loadRequired( + "swiftide_completion_result_foreach_baseexpr_typename" + ), + completion_result_is_reusing_astcontext: try loadRequired("swiftide_completion_result_is_reusing_astcontext"), + completion_item_description_copy: try loadRequired("swiftide_completion_item_description_copy"), + completion_item_get_label: try loadRequired("swiftide_completion_item_get_label"), + completion_item_get_source_text: try loadRequired("swiftide_completion_item_get_source_text"), + completion_item_get_type_name: try loadRequired("swiftide_completion_item_get_type_name"), + completion_item_get_doc_brief: try loadRequired("swiftide_completion_item_get_doc_brief"), + completion_item_get_associated_usrs: try loadRequired("swiftide_completion_item_get_associated_usrs"), + completion_item_get_kind: try loadRequired("swiftide_completion_item_get_kind"), + completion_item_get_associated_kind: try loadRequired("swiftide_completion_item_get_associated_kind"), + completion_item_get_semantic_context: try loadRequired("swiftide_completion_item_get_semantic_context"), + completion_item_get_flair: try loadRequired("swiftide_completion_item_get_flair"), + completion_item_is_not_recommended: try loadRequired("swiftide_completion_item_is_not_recommended"), + completion_item_not_recommended_reason: try loadRequired("swiftide_completion_item_not_recommended_reason"), + completion_item_has_diagnostic: try loadRequired("swiftide_completion_item_has_diagnostic"), + completion_item_get_diagnostic: try loadRequired("swiftide_completion_item_get_diagnostic"), + completion_item_is_system: try loadRequired("swiftide_completion_item_is_system"), + completion_item_get_module_name: try loadRequired("swiftide_completion_item_get_module_name"), + completion_item_get_num_bytes_to_erase: try loadRequired("swiftide_completion_item_get_num_bytes_to_erase"), + completion_item_get_type_relation: try loadRequired("swiftide_completion_item_get_type_relation"), + completion_item_import_depth: try loadRequired("swiftide_completion_item_import_depth"), + fuzzy_match_pattern_create: try loadRequired("swiftide_fuzzy_match_pattern_create"), + fuzzy_match_pattern_matches_candidate: try loadRequired("swiftide_fuzzy_match_pattern_matches_candidate"), + fuzzy_match_pattern_dispose: try loadRequired("swiftide_fuzzy_match_pattern_dispose") + ) + } +} + +extension sourcekitd_plugin_api_functions_t { + package init(_ sourcekitd: DLHandle) throws { + func loadRequired(_ symbol: String) throws -> T { + guard let sym: T = dlsym(sourcekitd, symbol: symbol) else { + throw SKDError.missingRequiredSymbol(symbol) + } + return sym + } + + func loadOptional(_ symbol: String) -> T? { + return dlsym(sourcekitd, symbol: symbol) + } + self.init( + variant_functions_create: try loadRequired("sourcekitd_variant_functions_create"), + variant_functions_set_get_type: try loadRequired("sourcekitd_variant_functions_set_get_type"), + variant_functions_set_array_apply: try loadRequired("sourcekitd_variant_functions_set_array_apply"), + variant_functions_set_array_get_bool: try loadRequired("sourcekitd_variant_functions_set_array_get_bool"), + variant_functions_set_array_get_double: try loadRequired("sourcekitd_variant_functions_set_array_get_double"), + variant_functions_set_array_get_count: try loadRequired("sourcekitd_variant_functions_set_array_get_count"), + variant_functions_set_array_get_int64: try loadRequired("sourcekitd_variant_functions_set_array_get_int64"), + variant_functions_set_array_get_string: try loadRequired("sourcekitd_variant_functions_set_array_get_string"), + variant_functions_set_array_get_uid: try loadRequired("sourcekitd_variant_functions_set_array_get_uid"), + variant_functions_set_array_get_value: try loadRequired("sourcekitd_variant_functions_set_array_get_value"), + variant_functions_set_bool_get_value: try loadRequired("sourcekitd_variant_functions_set_bool_get_value"), + variant_functions_set_double_get_value: try loadRequired("sourcekitd_variant_functions_set_double_get_value"), + variant_functions_set_dictionary_apply: try loadRequired("sourcekitd_variant_functions_set_dictionary_apply"), + variant_functions_set_dictionary_get_bool: try loadRequired( + "sourcekitd_variant_functions_set_dictionary_get_bool" + ), + variant_functions_set_dictionary_get_double: try loadRequired( + "sourcekitd_variant_functions_set_dictionary_get_double" + ), + variant_functions_set_dictionary_get_int64: try loadRequired( + "sourcekitd_variant_functions_set_dictionary_get_int64" + ), + variant_functions_set_dictionary_get_string: try loadRequired( + "sourcekitd_variant_functions_set_dictionary_get_string" + ), + variant_functions_set_dictionary_get_value: try loadRequired( + "sourcekitd_variant_functions_set_dictionary_get_value" + ), + variant_functions_set_dictionary_get_uid: try loadRequired("sourcekitd_variant_functions_set_dictionary_get_uid"), + variant_functions_set_string_get_length: try loadRequired("sourcekitd_variant_functions_set_string_get_length"), + variant_functions_set_string_get_ptr: try loadRequired("sourcekitd_variant_functions_set_string_get_ptr"), + variant_functions_set_int64_get_value: try loadRequired("sourcekitd_variant_functions_set_int64_get_value"), + variant_functions_set_uid_get_value: try loadRequired("sourcekitd_variant_functions_set_uid_get_value"), + variant_functions_set_data_get_size: try loadRequired("sourcekitd_variant_functions_set_data_get_size"), + variant_functions_set_data_get_ptr: try loadRequired("sourcekitd_variant_functions_set_data_get_ptr"), + plugin_initialize_is_client_only: try loadRequired("sourcekitd_plugin_initialize_is_client_only"), + plugin_initialize_custom_buffer_start: try loadRequired("sourcekitd_plugin_initialize_custom_buffer_start"), + plugin_initialize_uid_get_from_cstr: try loadRequired("sourcekitd_plugin_initialize_uid_get_from_cstr"), + plugin_initialize_uid_get_string_ptr: try loadRequired("sourcekitd_plugin_initialize_uid_get_string_ptr"), + plugin_initialize_register_custom_buffer: try loadRequired("sourcekitd_plugin_initialize_register_custom_buffer") + ) + } +} + +extension sourcekitd_service_plugin_api_functions_t { + package init(_ sourcekitd: DLHandle) throws { + func loadRequired(_ symbol: String) throws -> T { + guard let sym: T = dlsym(sourcekitd, symbol: symbol) else { + throw SKDError.missingRequiredSymbol(symbol) + } + return sym + } + + func loadOptional(_ symbol: String) -> T? { + return dlsym(sourcekitd, symbol: symbol) + } + self.init( + plugin_initialize_register_cancellable_request_handler: try loadRequired( + "sourcekitd_plugin_initialize_register_cancellable_request_handler" + ), + plugin_initialize_register_cancellation_handler: try loadRequired( + "sourcekitd_plugin_initialize_register_cancellation_handler" + ), + plugin_initialize_get_swift_ide_inspection_instance: try loadRequired( + "sourcekitd_plugin_initialize_get_swift_ide_inspection_instance" + ), + request_get_type: try loadRequired("sourcekitd_request_get_type"), + request_dictionary_get_value: try loadRequired("sourcekitd_request_dictionary_get_value"), + request_dictionary_get_string: try loadRequired("sourcekitd_request_dictionary_get_string"), + request_dictionary_get_int64: try loadRequired("sourcekitd_request_dictionary_get_int64"), + request_dictionary_get_bool: try loadRequired("sourcekitd_request_dictionary_get_bool"), + request_dictionary_get_uid: try loadRequired("sourcekitd_request_dictionary_get_uid"), + request_array_get_count: try loadRequired("sourcekitd_request_array_get_count"), + request_array_get_value: try loadRequired("sourcekitd_request_array_get_value"), + request_array_get_string: try loadRequired("sourcekitd_request_array_get_string"), + request_array_get_int64: try loadRequired("sourcekitd_request_array_get_int64"), + request_array_get_bool: try loadRequired("sourcekitd_request_array_get_bool"), + request_array_get_uid: try loadRequired("sourcekitd_request_array_get_uid"), + request_int64_get_value: try loadRequired("sourcekitd_request_int64_get_value"), + request_bool_get_value: try loadRequired("sourcekitd_request_bool_get_value"), + request_string_get_length: try loadRequired("sourcekitd_request_string_get_length"), + request_string_get_ptr: try loadRequired("sourcekitd_request_string_get_ptr"), + request_uid_get_value: try loadRequired("sourcekitd_request_uid_get_value"), + response_retain: try loadRequired("sourcekitd_response_retain"), + response_error_create: try loadRequired("sourcekitd_response_error_create"), + response_dictionary_create: try loadRequired("sourcekitd_response_dictionary_create"), + response_dictionary_set_value: try loadRequired("sourcekitd_response_dictionary_set_value"), + response_dictionary_set_string: try loadRequired("sourcekitd_response_dictionary_set_string"), + response_dictionary_set_stringbuf: try loadRequired("sourcekitd_response_dictionary_set_stringbuf"), + response_dictionary_set_int64: try loadRequired("sourcekitd_response_dictionary_set_int64"), + response_dictionary_set_bool: try loadRequired("sourcekitd_response_dictionary_set_bool"), + response_dictionary_set_double: try loadRequired("sourcekitd_response_dictionary_set_double"), + response_dictionary_set_uid: try loadRequired("sourcekitd_response_dictionary_set_uid"), + response_array_create: try loadRequired("sourcekitd_response_array_create"), + response_array_set_value: try loadRequired("sourcekitd_response_array_set_value"), + response_array_set_string: try loadRequired("sourcekitd_response_array_set_string"), + response_array_set_stringbuf: try loadRequired("sourcekitd_response_array_set_stringbuf"), + response_array_set_int64: try loadRequired("sourcekitd_response_array_set_int64"), + response_array_set_double: try loadRequired("sourcekitd_response_array_set_double"), + response_array_set_uid: try loadRequired("sourcekitd_response_array_set_uid"), + response_dictionary_set_custom_buffer: try loadRequired("sourcekitd_response_dictionary_set_custom_buffer") + ) + } +} diff --git a/Sources/SourceKitD/sourcekitd_uids.swift b/Sources/SourceKitD/sourcekitd_uids.swift index 4f2bb92c6..99e95827f 100644 --- a/Sources/SourceKitD/sourcekitd_uids.swift +++ b/Sources/SourceKitD/sourcekitd_uids.swift @@ -117,6 +117,10 @@ package struct sourcekitd_api_keys { package let numBytesToErase: sourcekitd_api_uid_t /// `key.not_recommended` package let notRecommended: sourcekitd_api_uid_t + /// `key.declarations` + package let declarations: sourcekitd_api_uid_t + /// `key.enabledeclarations` + package let enableDeclarations: sourcekitd_api_uid_t /// `key.annotations` package let annotations: sourcekitd_api_uid_t /// `key.semantic_tokens` @@ -463,6 +467,54 @@ package struct sourcekitd_api_keys { package let annotatedDescription: sourcekitd_api_uid_t /// `key.codecomplete.includeobjectliterals` package let includeObjectLiterals: sourcekitd_api_uid_t + /// `key.codecomplete.use_new_api` + package let useNewAPI: sourcekitd_api_uid_t + /// `key.codecomplete.addcallwithnodefaultargs` + package let addCallWithNoDefaultArgs: sourcekitd_api_uid_t + /// `key.codecomplete.include_semantic_components` + package let includeSemanticComponents: sourcekitd_api_uid_t + /// `key.codecomplete.use_xpc_serialization` + package let useXPCSerialization: sourcekitd_api_uid_t + /// `key.codecomplete.maxresults` + package let maxResults: sourcekitd_api_uid_t + /// `key.annotated.typename` + package let annotatedTypeName: sourcekitd_api_uid_t + /// `key.priority_bucket` + package let priorityBucket: sourcekitd_api_uid_t + /// `key.identifier` + package let identifier: sourcekitd_api_uid_t + /// `key.text_match_score` + package let textMatchScore: sourcekitd_api_uid_t + /// `key.semantic_score` + package let semanticScore: sourcekitd_api_uid_t + /// `key.semantic_score_components` + package let semanticScoreComponents: sourcekitd_api_uid_t + /// `key.symbol_popularity` + package let symbolPopularity: sourcekitd_api_uid_t + /// `key.module_popularity` + package let modulePopularity: sourcekitd_api_uid_t + /// `key.popularity.key` + package let popularityKey: sourcekitd_api_uid_t + /// `key.popularity.value.int.billion` + package let popularityValueIntBillion: sourcekitd_api_uid_t + /// `key.recent_completions` + package let recentCompletions: sourcekitd_api_uid_t + /// `key.unfiltered_result_count` + package let unfilteredResultCount: sourcekitd_api_uid_t + /// `key.member_access_types` + package let memberAccessTypes: sourcekitd_api_uid_t + /// `key.has_diagnostic` + package let hasDiagnostic: sourcekitd_api_uid_t + /// `key.group_id` + package let groupId: sourcekitd_api_uid_t + /// `key.scoped_popularity_table_path` + package let scopedPopularityTablePath: sourcekitd_api_uid_t + /// `key.popular_modules` + package let popularModules: sourcekitd_api_uid_t + /// `key.notorious_modules` + package let notoriousModules: sourcekitd_api_uid_t + /// `key.codecomplete.setpopularapi_used_score_components` + package let usedScoreComponents: sourcekitd_api_uid_t /// `key.editor.format.usetabs` package let useTabs: sourcekitd_api_uid_t /// `key.editor.format.indentwidth` @@ -522,6 +574,8 @@ package struct sourcekitd_api_keys { moduleImportDepth = api.uid_get_from_cstr("key.moduleimportdepth")! numBytesToErase = api.uid_get_from_cstr("key.num_bytes_to_erase")! notRecommended = api.uid_get_from_cstr("key.not_recommended")! + declarations = api.uid_get_from_cstr("key.declarations")! + enableDeclarations = api.uid_get_from_cstr("key.enabledeclarations")! annotations = api.uid_get_from_cstr("key.annotations")! semanticTokens = api.uid_get_from_cstr("key.semantic_tokens")! diagnosticStage = api.uid_get_from_cstr("key.diagnostic_stage")! @@ -695,6 +749,30 @@ package struct sourcekitd_api_keys { popularityBonus = api.uid_get_from_cstr("key.codecomplete.sort.popularitybonus")! annotatedDescription = api.uid_get_from_cstr("key.codecomplete.annotateddescription")! includeObjectLiterals = api.uid_get_from_cstr("key.codecomplete.includeobjectliterals")! + useNewAPI = api.uid_get_from_cstr("key.codecomplete.use_new_api")! + addCallWithNoDefaultArgs = api.uid_get_from_cstr("key.codecomplete.addcallwithnodefaultargs")! + includeSemanticComponents = api.uid_get_from_cstr("key.codecomplete.include_semantic_components")! + useXPCSerialization = api.uid_get_from_cstr("key.codecomplete.use_xpc_serialization")! + maxResults = api.uid_get_from_cstr("key.codecomplete.maxresults")! + annotatedTypeName = api.uid_get_from_cstr("key.annotated.typename")! + priorityBucket = api.uid_get_from_cstr("key.priority_bucket")! + identifier = api.uid_get_from_cstr("key.identifier")! + textMatchScore = api.uid_get_from_cstr("key.text_match_score")! + semanticScore = api.uid_get_from_cstr("key.semantic_score")! + semanticScoreComponents = api.uid_get_from_cstr("key.semantic_score_components")! + symbolPopularity = api.uid_get_from_cstr("key.symbol_popularity")! + modulePopularity = api.uid_get_from_cstr("key.module_popularity")! + popularityKey = api.uid_get_from_cstr("key.popularity.key")! + popularityValueIntBillion = api.uid_get_from_cstr("key.popularity.value.int.billion")! + recentCompletions = api.uid_get_from_cstr("key.recent_completions")! + unfilteredResultCount = api.uid_get_from_cstr("key.unfiltered_result_count")! + memberAccessTypes = api.uid_get_from_cstr("key.member_access_types")! + hasDiagnostic = api.uid_get_from_cstr("key.has_diagnostic")! + groupId = api.uid_get_from_cstr("key.group_id")! + scopedPopularityTablePath = api.uid_get_from_cstr("key.scoped_popularity_table_path")! + popularModules = api.uid_get_from_cstr("key.popular_modules")! + notoriousModules = api.uid_get_from_cstr("key.notorious_modules")! + usedScoreComponents = api.uid_get_from_cstr("key.codecomplete.setpopularapi_used_score_components")! useTabs = api.uid_get_from_cstr("key.editor.format.usetabs")! indentWidth = api.uid_get_from_cstr("key.editor.format.indentwidth")! tabWidth = api.uid_get_from_cstr("key.editor.format.tabwidth")! @@ -809,6 +887,10 @@ package struct sourcekitd_api_requests { package let syntacticMacroExpansion: sourcekitd_api_uid_t /// `source.request.index_to_store` package let indexToStore: sourcekitd_api_uid_t + /// `source.request.codecomplete.documentation` + package let codeCompleteDocumentation: sourcekitd_api_uid_t + /// `source.request.codecomplete.diagnostic` + package let codeCompleteDiagnostic: sourcekitd_api_uid_t package init(api: sourcekitd_api_functions_t) { protocolVersion = api.uid_get_from_cstr("source.request.protocol_version")! @@ -864,6 +946,8 @@ package struct sourcekitd_api_requests { enableRequestBarriers = api.uid_get_from_cstr("source.request.enable_request_barriers")! syntacticMacroExpansion = api.uid_get_from_cstr("source.request.syntactic_macro_expansion")! indexToStore = api.uid_get_from_cstr("source.request.index_to_store")! + codeCompleteDocumentation = api.uid_get_from_cstr("source.request.codecomplete.documentation")! + codeCompleteDiagnostic = api.uid_get_from_cstr("source.request.codecomplete.diagnostic")! } } @@ -1248,6 +1332,8 @@ package struct sourcekitd_api_values { package let semaEnabledNotification: sourcekitd_api_uid_t /// `source.notification.editor.documentupdate` package let documentUpdateNotification: sourcekitd_api_uid_t + /// `source.diagnostic.severity.remark` + package let diagRemark: sourcekitd_api_uid_t package init(api: sourcekitd_api_functions_t) { declFunctionFree = api.uid_get_from_cstr("source.lang.swift.decl.function.free")! @@ -1440,5 +1526,6 @@ package struct sourcekitd_api_values { semaDisabledNotification = api.uid_get_from_cstr("source.notification.sema_disabled")! semaEnabledNotification = api.uid_get_from_cstr("source.notification.sema_enabled")! documentUpdateNotification = api.uid_get_from_cstr("source.notification.editor.documentupdate")! + diagRemark = api.uid_get_from_cstr("source.diagnostic.severity.remark")! } } diff --git a/Sources/SourceKitD/sourcekitd_uids.swift.gyb b/Sources/SourceKitD/sourcekitd_uids.swift.gyb index be92a3774..f1a5263c9 100644 --- a/Sources/SourceKitD/sourcekitd_uids.swift.gyb +++ b/Sources/SourceKitD/sourcekitd_uids.swift.gyb @@ -31,7 +31,6 @@ return name return name[:word_index].lower() + name[word_index:] - # We should automatically generate these. ADDITIONAL_KEYS = [ # Maintained from translateCodeCompletionOptions in SwiftCompletion.cpp# KEY('SortByName', 'key.codecomplete.sort.byname'), @@ -56,6 +55,33 @@ KEY('AnnotatedDescription', 'key.codecomplete.annotateddescription'), KEY('IncludeObjectLiterals', 'key.codecomplete.includeobjectliterals'), + # Used exclusively within the SourceKit Plugin + KEY('UseNewAPI', 'key.codecomplete.use_new_api'), + KEY('AddCallWithNoDefaultArgs', 'key.codecomplete.addcallwithnodefaultargs'), + KEY('IncludeSemanticComponents', 'key.codecomplete.include_semantic_components'), + KEY('UseXPCSerialization', 'key.codecomplete.use_xpc_serialization'), + KEY('MaxResults', 'key.codecomplete.maxresults'), + KEY('AnnotatedTypeName', 'key.annotated.typename'), + KEY('PriorityBucket', 'key.priority_bucket'), + KEY('Identifier', 'key.identifier'), + KEY('TextMatchScore', 'key.text_match_score'), + KEY('SemanticScore', 'key.semantic_score'), + KEY('SemanticScoreComponents', 'key.semantic_score_components'), + KEY('SymbolPopularity', 'key.symbol_popularity'), + KEY('ModulePopularity', 'key.module_popularity'), + KEY('PopularityKey', 'key.popularity.key'), + KEY('PopularityValueIntBillion', 'key.popularity.value.int.billion'), + KEY('RecentCompletions', 'key.recent_completions'), + KEY('UnfilteredResultCount', 'key.unfiltered_result_count'), + KEY('MemberAccessTypes', 'key.member_access_types'), + KEY('HasDiagnostic', 'key.has_diagnostic'), + KEY('GroupId', 'key.group_id'), + KEY('ScopedPopularityTablePath', 'key.scoped_popularity_table_path'), + KEY('PopularModules', 'key.popular_modules'), + KEY('NotoriousModules', 'key.notorious_modules'), + KEY('UsedScoreComponents', 'key.codecomplete.setpopularapi_used_score_components'), + + # Maintained from applyFormatOptions in SwiftEditor.cpp KEY('UseTabs', 'key.editor.format.usetabs'), KEY('IndentWidth', 'key.editor.format.indentwidth'), @@ -63,6 +89,11 @@ KEY('IndentSwitchCase', 'key.editor.format.indent_switch_case'), ] + ADDITIONAL_REQUESTS = [ + REQUEST('CodeCompleteDocumentation', 'source.request.codecomplete.documentation'), + REQUEST('CodeCompleteDiagnostic', 'source.request.codecomplete.diagnostic'), + ] + # We should automatically generate these. ADDITIONAL_VALUES = [ # Maintained from SwiftToSourceKitCompletionAdapter::handleResult in SwiftCompletion.cpp @@ -79,16 +110,23 @@ # Maintained from initializeService in Requests.cpp KIND('SemaEnabledNotification', 'source.notification.sema_enabled'), KIND('DocumentUpdateNotification', 'source.notification.editor.documentupdate'), + + # Used exclusively within the SourceKit Plugin + KIND('DiagRemark', 'source.diagnostic.severity.remark'), ] TYPES_AND_KEYS = [ ('sourcekitd_api_keys', UID_KEYS + ADDITIONAL_KEYS), - ('sourcekitd_api_requests', UID_REQUESTS), + ('sourcekitd_api_requests', UID_REQUESTS + ADDITIONAL_REQUESTS), ('sourcekitd_api_values', UID_KINDS + ADDITIONAL_VALUES), ] }% +#if compiler(>=6) +package import Csourcekitd +#else import Csourcekitd +#endif % for (struct_type, uids) in TYPES_AND_KEYS: package struct ${struct_type} { diff --git a/Sources/SourceKitDForPlugin b/Sources/SourceKitDForPlugin new file mode 120000 index 000000000..9c224c633 --- /dev/null +++ b/Sources/SourceKitDForPlugin @@ -0,0 +1 @@ +SourceKitD \ No newline at end of file diff --git a/Sources/SwiftExtensionsForPlugin b/Sources/SwiftExtensionsForPlugin new file mode 120000 index 000000000..6f5bb646e --- /dev/null +++ b/Sources/SwiftExtensionsForPlugin @@ -0,0 +1 @@ +SwiftExtensions \ No newline at end of file diff --git a/Sources/SwiftSourceKitClientPlugin/ClientPlugin.swift b/Sources/SwiftSourceKitClientPlugin/ClientPlugin.swift new file mode 100644 index 000000000..06a115c72 --- /dev/null +++ b/Sources/SwiftSourceKitClientPlugin/ClientPlugin.swift @@ -0,0 +1,32 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import SourceKitD +import SwiftSourceKitPluginCommon + +#if compiler(>=6) +public import Csourcekitd +#else +import Csourcekitd +#endif + +@_cdecl("sourcekitd_plugin_initialize") +public func sourcekitd_plugin_initialize(_ params: sourcekitd_api_plugin_initialize_params_t) { + let skd = DynamicallyLoadedSourceKitD.relativeToPlugin + let customBufferStart = skd.pluginApi.plugin_initialize_custom_buffer_start(params) + let arrayBuffKind = customBufferStart + skd.pluginApi.plugin_initialize_register_custom_buffer( + params, + arrayBuffKind, + CompletionResultsArray.arrayFuncs.rawValue + ) +} diff --git a/Sources/SwiftSourceKitPlugin/ASTCompletion/ASTCompletionItem.swift b/Sources/SwiftSourceKitPlugin/ASTCompletion/ASTCompletionItem.swift new file mode 100644 index 000000000..e92a5c3af --- /dev/null +++ b/Sources/SwiftSourceKitPlugin/ASTCompletion/ASTCompletionItem.swift @@ -0,0 +1,647 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import CompletionScoring +import Csourcekitd +import Foundation +import SourceKitD + +/// A value that is computed on its first access and saved for later retrievals. +enum LazyValue { + case computed(T) + case uninitialized + + /// If the value has already been computed return it, otherwise compute it using `compute`. + mutating func cachedValueOrCompute(_ compute: () -> T) -> T { + switch self { + case .computed(let value): + return value + case .uninitialized: + let newValue = compute() + self = .computed(newValue) + return newValue + } + } +} + +/// A single code completion result returned from sourcekitd + additional information. This is effectively a wrapper +/// around `swiftide_api_completion_item_t` that caches the properties which have already been retrieved. +/// - Note: There can many be `ASTCompletionItem` instances (e.g. global completion has ~100k items), make sure to check +/// layout when adding new fields to ensure we're not wasting a bunch of space. +/// (i.e.,`heap --showInternalFragmentation process_name`) +final class ASTCompletionItem { + let impl: swiftide_api_completion_item_t + + /// The string that should be used to match against what the user type. + var filterName: String { + return _filterName.cachedValueOrCompute { + filterNameCString != nil ? String(cString: filterNameCString!) : "" + } + } + let filterNameCString: UnsafePointer? + private var _filterName: LazyValue = .uninitialized + + /// The label with which the item should be displayed in an IDE + func label(in session: CompletionSession) -> String { + return _label.cachedValueOrCompute { + var value: String? + session.sourcekitd.ideApi.completion_item_get_label(session.response, impl, session.options.annotateResults) { + value = String(cString: $0!) + } + return value! + } + } + private var _label: LazyValue = .uninitialized + + func sourceText(in session: CompletionSession) -> String { + return _sourceText.cachedValueOrCompute { + var value: String? + session.sourcekitd.ideApi.completion_item_get_source_text(session.response, impl) { + value = String(cString: $0!) + } + return value! + } + } + private var _sourceText: LazyValue = .uninitialized + + /// The type that the code completion item produces. + /// + /// Eg. the type of a variable or the return type of a function. `nil` for completions that don't have a type, like + /// keywords. + func typeName(in session: CompletionSession) -> String? { + return _typeName.cachedValueOrCompute { + var value: String? + session.sourcekitd.ideApi.completion_item_get_type_name(session.response, impl, session.options.annotateResults) { + if let cstr = $0 { + value = String(cString: cstr) + } + } + return value + } + } + private var _typeName: LazyValue = .uninitialized + + /// The module that defines the code completion item or `nil` if the item is not defined in a module, like a keyword. + func moduleName(in session: CompletionSession) -> String? { + return _moduleName.cachedValueOrCompute { + var value: String? + session.sourcekitd.ideApi.completion_item_get_module_name(session.response, impl) { + if let cstr = $0 { + value = String(cString: cstr) + } else { + value = nil + } + } + if value == "" { + return nil + } + return value + } + } + private var _moduleName: LazyValue = .uninitialized + + func priorityBucket(in session: CompletionSession) -> CompletionItem.PriorityBucket { + return _priorityBucket.cachedValueOrCompute { + CompletionItem.PriorityBucket(self, in: session) + } + } + private var _priorityBucket: LazyValue = .uninitialized + + let completionKind: CompletionContext.Kind + + let index: UInt32 + + func semanticScore(in session: CompletionSession) -> Double { + return _semanticScore.cachedValueOrCompute { + let semanticClassification = semanticClassification(in: session) + self.semanticClassification = semanticClassification + return semanticClassification.score + } + } + private var _semanticScore: LazyValue = .uninitialized + + private func semanticClassification(in session: CompletionSession) -> SemanticClassification { + var module = moduleName(in: session) + if let baseModule = module?.split(separator: ".", maxSplits: 1).first { + // `PopularityIndex` is keyed on base module names. + // For example: "AppKit.NSImage" -> "AppKit". + module = String(baseModule) + } + let popularity = session.popularity( + ofSymbol: filterName, + inModule: module + ) + return SemanticClassification( + availability: availability(in: session), + completionKind: semanticScoreCompletionKind(in: session), + flair: flair(in: session), + moduleProximity: moduleProximity(in: session), + popularity: popularity ?? .none, + scopeProximity: scopeProximity(in: session), + structuralProximity: structuralProximity(in: session), + synchronicityCompatibility: synchronicityCompatibility(in: session), + typeCompatibility: typeCompatibility(in: session) + ) + } + var semanticClassification: SemanticClassification? = nil + + var kind: CompletionItem.ItemKind + + func semanticContext(in session: CompletionSession) -> CompletionItem.SemanticContext { + .init( + swiftide_api_completion_semantic_context_t(session.sourcekitd.ideApi.completion_item_get_semantic_context(impl)) + ) + } + + func typeRelation(in session: CompletionSession) -> CompletionItem.TypeRelation { + .init(swiftide_api_completion_type_relation_t(session.sourcekitd.ideApi.completion_item_get_type_relation(impl))) + } + + func numBytesToErase(in session: CompletionSession) -> Int { + Int(session.sourcekitd.ideApi.completion_item_get_num_bytes_to_erase(impl)) + } + + func notRecommended(in session: CompletionSession) -> Bool { + session.sourcekitd.ideApi.completion_item_is_not_recommended(impl) + } + + func notRecommendedReason(in session: CompletionSession) -> NotRecommendedReason? { + guard notRecommended(in: session) else { + return nil + } + return NotRecommendedReason(impl, sourcekitd: session.sourcekitd) + } + + func isSystem(in session: CompletionSession) -> Bool { session.sourcekitd.ideApi.completion_item_is_system(impl) } + + func hasDiagnostic(in session: CompletionSession) -> Bool { + session.sourcekitd.ideApi.completion_item_has_diagnostic(impl) + } + + init( + _ cresult: swiftide_api_completion_item_t, + filterName: UnsafePointer?, + completionKind: CompletionContext.Kind, + index: UInt32, + sourcekitd: any SourceKitD + ) { + self.impl = cresult + self.filterNameCString = filterName + self.completionKind = completionKind + self.index = index + self.kind = .init( + swiftide_api_completion_item_kind_t(sourcekitd.ideApi.completion_item_get_kind(impl)), + associatedKind: sourcekitd.ideApi.completion_item_get_associated_kind(impl) + ) + } + + enum NotRecommendedReason { + case softDeprecated + case deprecated + case redundantImport + case redundantImportImplicit + case invalidAsyncContext + case crossActorReference + case variableUsedInOwnDefinition + case nonAsyncAlternativeUsedInAsyncContext + + init?(_ item: swiftide_api_completion_item_t, sourcekitd: SourceKitD) { + let rawReason = sourcekitd.ideApi.completion_item_not_recommended_reason(item) + switch swiftide_api_completion_not_recommended_reason_t(rawReason) { + case SWIFTIDE_COMPLETION_NOT_RECOMMENDED_NONE: + return nil + case SWIFTIDE_COMPLETION_NOT_RECOMMENDED_REDUNDANT_IMPORT: + self = .redundantImport + case SWIFTIDE_COMPLETION_NOT_RECOMMENDED_DEPRECATED: + self = .deprecated + case SWIFTIDE_COMPLETION_NOT_RECOMMENDED_INVALID_ASYNC_CONTEXT: + self = .invalidAsyncContext + case SWIFTIDE_COMPLETION_NOT_RECOMMENDED_CROSS_ACTOR_REFERENCE: + self = .crossActorReference + case SWIFTIDE_COMPLETION_NOT_RECOMMENDED_VARIABLE_USED_IN_OWN_DEFINITION: + self = .variableUsedInOwnDefinition + case SWIFTIDE_COMPLETION_NOT_RECOMMENDED_SOFTDEPRECATED: + self = .softDeprecated + case SWIFTIDE_COMPLETION_NOT_RECOMMENDED_REDUNDANT_IMPORT_INDIRECT: + self = .redundantImportImplicit + case SWIFTIDE_COMPLETION_NOT_RECOMMENDED_NON_ASYNC_ALTERNATIVE_USED_IN_ASYNC_CONTEXT: + self = .nonAsyncAlternativeUsedInAsyncContext + default: + return nil + } + } + } +} + +extension ASTCompletionItem { + private func semanticScoreCompletionKind(in session: CompletionSession) -> CompletionKind { + if session.sourcekitd.ideApi.completion_item_get_flair(impl) & SWIFTIDE_COMPLETION_FLAIR_ARGUMENTLABELS.rawValue + != 0 + { + return .argumentLabels + } + switch kind { + case .module: + return .module + case .class, .actor, .struct, .enum, .protocol, .associatedType, .typeAlias, .genericTypeParam, .precedenceGroup: + return .type + case .enumElement: + return .enumCase + case .constructor: + return .initializer + case .destructor: + // FIXME: add a "deinit" kind. + return .function + case .subscript: + // FIXME: add a "subscript" kind. + return .function + case .staticMethod, .instanceMethod, .freeFunction: + return .function + case .operator, .prefixOperatorFunction, .postfixOperatorFunction, .infixOperatorFunction: + // FIXME: add an "operator kind". + return .other + case .staticVar, .instanceVar, .localVar, .globalVar: + return .variable + case .keyword: + return .keyword + case .literal: + // FIXME: add a "literal" kind? + return .other + case .pattern: + // FIXME: figure out a kind for this. + return .other + case .macro: + // FIXME: add a "macro" kind? + return .type + case .unknown: + return .unknown + } + } + + private func flair(in session: CompletionSession) -> Flair { + var result: Flair = [] + let skFlair = session.sourcekitd.ideApi.completion_item_get_flair(impl) + if skFlair & SWIFTIDE_COMPLETION_FLAIR_EXPRESSIONSPECIFIC.rawValue != 0 { + result.insert(.oldExpressionSpecific_pleaseAddSpecificCaseToThisEnum) + } + if skFlair & SWIFTIDE_COMPLETION_FLAIR_SUPERCHAIN.rawValue != 0 { + result.insert(.chainedCallToSuper) + } + if skFlair & SWIFTIDE_COMPLETION_FLAIR_COMMONKEYWORDATCURRENTPOSITION.rawValue != 0 { + result.insert(.commonKeywordAtCurrentPosition) + } + if skFlair & SWIFTIDE_COMPLETION_FLAIR_RAREKEYWORDATCURRENTPOSITION.rawValue != 0 { + result.insert(.rareKeywordAtCurrentPosition) + } + if skFlair & SWIFTIDE_COMPLETION_FLAIR_RARETYPEATCURRENTPOSITION.rawValue != 0 { + result.insert(.rareKeywordAtCurrentPosition) + } + if skFlair & SWIFTIDE_COMPLETION_FLAIR_EXPRESSIONATNONSCRIPTORMAINFILESCOPE.rawValue != 0 { + result.insert(.expressionAtNonScriptOrMainFileScope) + } + return result + } + + private func moduleProximity(in session: CompletionSession) -> ModuleProximity { + switch semanticContext(in: session) { + case .none: + return .inapplicable + case .local, .currentNominal, .outsideNominal: + return .imported(distance: 0) + case .super: + // FIXME: we don't know whether the super class is from this module or another. + return .unspecified + case .currentModule: + return .imported(distance: 0) + case .otherModule: + let depth = session.sourcekitd.ideApi.completion_item_import_depth(session.response, self.impl) + if depth == ~0 { + return .unknown + } else { + return .imported(distance: Int(depth)) + } + } + } + + private func scopeProximity(in session: CompletionSession) -> ScopeProximity { + switch semanticContext(in: session) { + case .none: + return .inapplicable + case .local: + return .local + case .currentNominal: + return .container + case .super: + return .inheritedContainer + case .outsideNominal: + return .outerContainer + case .currentModule, .otherModule: + return .global + } + } + + private func structuralProximity(in session: CompletionSession) -> StructuralProximity { + switch kind { + case .keyword, .literal: + return .inapplicable + default: + return isSystem(in: session) ? .sdk : .project(fileSystemHops: nil) + } + } + + func synchronicityCompatibility(in session: CompletionSession) -> SynchronicityCompatibility { + return notRecommendedReason(in: session) == .invalidAsyncContext ? .incompatible : .compatible + } + + func typeCompatibility(in session: CompletionSession) -> TypeCompatibility { + switch typeRelation(in: session) { + case .identical: return .compatible + case .convertible: return .compatible + case .notApplicable: return .inapplicable + case .unrelated: return .unrelated + // Note: currently `unknown` in sourcekit usually means there is no context (e.g. statement level), which is + // equivalent to `inapplicable`. For now, map it that way to avoid spurious penalties. + case .unknown: return .inapplicable + case .invalid: return .invalid + } + } + + func availability(in session: CompletionSession) -> Availability { + switch notRecommendedReason(in: session) { + case .deprecated: + return .deprecated + case .softDeprecated: + return .softDeprecated + case .invalidAsyncContext, .crossActorReference: + return .available + case .redundantImport, .variableUsedInOwnDefinition: + return .softDeprecated + case .redundantImportImplicit: + return .available + case .nonAsyncAlternativeUsedInAsyncContext: + return .softDeprecated + case nil: + return .available + } + } +} + +extension CompletionItem { + init( + _ astItem: ASTCompletionItem, + score: CompletionScore, + in session: CompletionSession, + completionReplaceRange: Range, + groupID: (_ baseName: String) -> Int + ) { + self.label = astItem.label(in: session) + self.filterText = astItem.filterName + self.module = astItem.moduleName(in: session) + self.typeName = astItem.typeName(in: session) + var editRange = completionReplaceRange + if astItem.numBytesToErase(in: session) > 0 { + var newCol = editRange.lowerBound.utf8Column - astItem.numBytesToErase(in: session) + if newCol < 1 { + assertionFailure("num_bytes_to_erase crosses line boundary") + newCol = 1 + } + editRange = Position(line: editRange.lowerBound.line, utf8Column: newCol).. [CompletionItem] { + let sorting = CompletionSorting(filterText: filterText, in: self) + let range = + location.position.. [CompletionItem] in + var nextGroupId = 1 // NOTE: Never use zero. 0 can be considered null groupID. + var baseNameToGroupId: [String: Int] = [:] + + return matches.map { + CompletionItem( + items[$0.index], + score: $0.score, + in: self, + completionReplaceRange: range, + groupID: { (baseName: String) -> Int in + if let entry = baseNameToGroupId[baseName] { + return entry + } else { + let groupId = nextGroupId + baseNameToGroupId[baseName] = groupId + nextGroupId += 1 + return groupId + } + } + ) + } + } + } + + deinit { + sourcekitd.ideApi.completion_result_dispose(response) + } + + func popularity(ofSymbol name: String, inModule module: String?) -> Popularity? { + guard let popularityIndex = self.popularityIndex else { + // Fall back to deprecated 'popularityTable'. + if let popularityTable = self.popularityTable { + return popularityTable.popularity(symbol: name, module: module) + } + + return nil + } + + let shouldUseBaseExprScope: Bool + // Use the base expression scope, for member completions. + switch completionKind { + case .dotExpr, .unresolvedMember, .postfixExpr, .keyPathExprSwift, .keyPathExprObjC: + shouldUseBaseExprScope = true + default: + // FIXME: 'baseExprScope' might still be populated for implicit self + // members. e.g. global expression completion in a method. + // We might want to use `baseExprScope` if the symbol is a type member. + shouldUseBaseExprScope = false + } + + let scope: PopularityIndex.Scope + // 'baseExprScope == nil' means the 'PopularityIndex' doesn't know the scope. + // Fallback to the symbol module scope. + if shouldUseBaseExprScope, let baseExprScope = context.baseExprScope { + scope = baseExprScope + } else { + guard let module = module else { + // Keywords, etc. don't belong to any module. + return nil + } + scope = PopularityIndex.Scope(container: nil, module: module) + } + + // Extract the base name from 'name'. + let baseName: String + if let parenIdx = name.firstIndex(of: "(") { + baseName = String(name[.. ExtendedCompletionInfo? { + return ExtendedCompletionInfo(session: self, index: Int(id.index)) + } + + var completionKind: CompletionContext.Kind { context.kind } + var memberAccessTypes: [String] { context.memberAccessTypes } +} + +/// Information about code completion items that is not returned to the client with the initial results but that the +/// client needs to request for each item with a separate request. It is intended that the client only requests this +/// information when more information about a code completion items should be displayed, eg. because the user selected +/// it. +struct ExtendedCompletionInfo { + private let session: CompletionSession + + /// The index of the item to get extended information for in `session.items`. + private let index: Int + + private var rawItem: swiftide_api_completion_item_t { session.items[index].impl } + + init(session: CompletionSession, index: Int) { + self.session = session + self.index = index + } + + var briefDocumentation: String? { + var result: String? = nil + session.sourcekitd.ideApi.completion_item_get_doc_brief(session.response, rawItem) { + if let cstr = $0 { + result = String(cString: cstr) + } + } + return result + } + + var associatedUSRs: [String] { + var result: [String] = [] + session.sourcekitd.ideApi.completion_item_get_associated_usrs(session.response, rawItem) { ptr, len in + result.reserveCapacity(Int(len)) + for usr in UnsafeBufferPointer(start: ptr, count: Int(len)) { + if let cstr = usr { + result.append(String(cString: cstr)) + } + } + } + return result + } + + var diagnostic: CompletionItem.Diagnostic? { + var result: CompletionItem.Diagnostic? = nil + session.sourcekitd.ideApi.completion_item_get_diagnostic(session.response, rawItem) { severity, message in + if let severity = CompletionItem.Diagnostic.Severity(severity) { + result = .init(severity: severity, description: String(cString: message!)) + } + } + return result + } +} + +extension CompletionItem.Diagnostic.Severity { + init?(_ ideValue: swiftide_api_completion_diagnostic_severity_t) { + switch ideValue { + case SWIFTIDE_COMPLETION_DIAGNOSTIC_SEVERITY_ERROR: + self = .error + case SWIFTIDE_COMPLETION_DIAGNOSTIC_SEVERITY_WARNING: + self = .warning + case SWIFTIDE_COMPLETION_DIAGNOSTIC_SEVERITY_REMARK: + self = .remark + case SWIFTIDE_COMPLETION_DIAGNOSTIC_SEVERITY_NOTE: + self = .note + case SWIFTIDE_COMPLETION_DIAGNOSTIC_SEVERITY_NONE: + return nil + default: + // FIXME: Handle unknown severity? + return nil + } + } +} diff --git a/Sources/SwiftSourceKitPlugin/CodeCompletion/Completion+Diagnostic.swift b/Sources/SwiftSourceKitPlugin/CodeCompletion/Completion+Diagnostic.swift new file mode 100644 index 000000000..6c336226b --- /dev/null +++ b/Sources/SwiftSourceKitPlugin/CodeCompletion/Completion+Diagnostic.swift @@ -0,0 +1,32 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +extension CompletionItem { + /// A diagnostic associated with a given completion, for example, because it + /// is a completion for a deprecated declaration. + struct Diagnostic { + enum Severity { + case note + case remark + case warning + case error + } + + var severity: Severity + var description: String + + init(severity: Severity, description: String) { + self.severity = severity + self.description = description + } + } +} diff --git a/Sources/SwiftSourceKitPlugin/CodeCompletion/Completion+Identifier.swift b/Sources/SwiftSourceKitPlugin/CodeCompletion/Completion+Identifier.swift new file mode 100644 index 000000000..8630e27a5 --- /dev/null +++ b/Sources/SwiftSourceKitPlugin/CodeCompletion/Completion+Identifier.swift @@ -0,0 +1,33 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +extension CompletionItem { + /// A unique identifier for the completion within a given session. + struct Identifier: Hashable { + /// The index of this completion item within the code completion session. + let index: UInt32 + + init(index: UInt32) { + self.index = index + } + + /// Restore an `Identifier` from a value retrieved from `opaqueValue`. + init(opaqueValue: Int64) { + self.init(index: UInt32(bitPattern: Int32(opaqueValue))) + } + + /// Representation of this identifier as an `Int64`, which can be transferred in sourcekitd requests. + var opaqueValue: Int64 { + Int64(bitPattern: UInt64(index)) + } + } +} diff --git a/Sources/SwiftSourceKitPlugin/CodeCompletion/Completion+ItemKind.swift b/Sources/SwiftSourceKitPlugin/CodeCompletion/Completion+ItemKind.swift new file mode 100644 index 000000000..531427063 --- /dev/null +++ b/Sources/SwiftSourceKitPlugin/CodeCompletion/Completion+ItemKind.swift @@ -0,0 +1,48 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +extension CompletionItem { + enum ItemKind { + // Decls + case module + case `class` + case actor + case `struct` + case `enum` + case enumElement + case `protocol` + case associatedType + case typeAlias + case genericTypeParam + case constructor + case destructor + case `subscript` + case staticMethod + case instanceMethod + case prefixOperatorFunction + case postfixOperatorFunction + case infixOperatorFunction + case freeFunction + case staticVar + case instanceVar + case localVar + case globalVar + case precedenceGroup + // Other + case keyword + case `operator` + case literal + case pattern + case macro + case unknown + } +} diff --git a/Sources/SwiftSourceKitPlugin/CodeCompletion/Completion+PriorityBucket.swift b/Sources/SwiftSourceKitPlugin/CodeCompletion/Completion+PriorityBucket.swift new file mode 100644 index 000000000..d864595a4 --- /dev/null +++ b/Sources/SwiftSourceKitPlugin/CodeCompletion/Completion+PriorityBucket.swift @@ -0,0 +1,69 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +extension CompletionItem { + struct PriorityBucket: RawRepresentable, Comparable { + var rawValue: Int + + init(rawValue: Int) { + self.rawValue = rawValue + } + + static let userPrioritized: PriorityBucket = .init(rawValue: 1) + static let highlyLikely: PriorityBucket = .init(rawValue: 5) + static let likely: PriorityBucket = .init(rawValue: 10) + static let regular: PriorityBucket = .init(rawValue: 50) + static let wordsInFile: PriorityBucket = .init(rawValue: 90) + static let infrequentlyUsed: PriorityBucket = .init(rawValue: 100) + static let unknown: PriorityBucket = .init(rawValue: .max) + + // Swift Semantic Entries + static let unresolvedMember_EnumElement: PriorityBucket = .highlyLikely + -4 + static let unresolvedMember_Var: PriorityBucket = .highlyLikely + -3 + static let unresolvedMember_Func: PriorityBucket = .highlyLikely + -2 + static let unresolvedMember_Constructor: PriorityBucket = .highlyLikely + -1 + static let unresolvedMember_Other: PriorityBucket = .regular + 0 + static let constructor: PriorityBucket = .highlyLikely + 0 + static let invalidTypeMatch: PriorityBucket = .infrequentlyUsed + 0 + static let otherModule_TypeMatch: PriorityBucket = .likely + 0 + static let otherModule_TypeMismatch: PriorityBucket = .regular + 0 + static let thisModule_TypeMatch: PriorityBucket = .likely + -1 + static let thisModule_TypeMismatch: PriorityBucket = .regular + -1 + static let noContext_TypeMatch: PriorityBucket = .likely + 0 + static let noContext_TypeMismatch: PriorityBucket = .regular + 0 + static let superClass_TypeMatch: PriorityBucket = .likely + -3 + static let superClass_TypeMismatch: PriorityBucket = .likely + 0 + static let thisClass_TypeMatch: PriorityBucket = .likely + -4 + static let thisClass_TypeMismatch: PriorityBucket = .likely + -1 + static let local_TypeMatch: PriorityBucket = .highlyLikely + 0 + static let local_TypeMismatch: PriorityBucket = .likely + -2 + static let otherClass_TypeMatch: PriorityBucket = .highlyLikely + 0 + static let otherClass_TypeMismatch: PriorityBucket = .likely + 0 + static let exprSpecific: PriorityBucket = .highlyLikely + 0 + + var scoreCoefficient: Double { + let clipped = max(min(self.rawValue, 100), 0) + let v = Double(100 - clipped) / 100.0 + return 1.0 + v * v * v + } + + static func + (lhs: PriorityBucket, rhs: Int) -> PriorityBucket { + return PriorityBucket(rawValue: lhs.rawValue + rhs) + } + static func - (lhs: PriorityBucket, rhs: Int) -> PriorityBucket { + return PriorityBucket(rawValue: lhs.rawValue - rhs) + } + static func < (lhs: PriorityBucket, rhs: PriorityBucket) -> Bool { + return lhs.rawValue < rhs.rawValue + } + } +} diff --git a/Sources/SwiftSourceKitPlugin/CodeCompletion/Completion+SemanticContext.swift b/Sources/SwiftSourceKitPlugin/CodeCompletion/Completion+SemanticContext.swift new file mode 100644 index 000000000..c981535dd --- /dev/null +++ b/Sources/SwiftSourceKitPlugin/CodeCompletion/Completion+SemanticContext.swift @@ -0,0 +1,49 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +extension CompletionItem { + enum SemanticContext { + /// Used in cases when the concept of semantic context is not applicable. + case none + + /// A declaration from the same function. + case local + + /// A declaration found in the immediately enclosing nominal decl. + case currentNominal + + /// A declaration found in the superclass of the immediately enclosing + /// nominal decl. + case `super` + + /// A declaration found in the non-immediately enclosing nominal decl. + /// + /// For example, 'Foo' is visible at (1) because of this. + /// ``` + /// struct A { + /// typealias Foo = Int + /// struct B { + /// func foo() { + /// // (1) + /// } + /// } + /// } + /// ``` + case outsideNominal + + /// A declaration from the current module. + case currentModule + + /// A declaration imported from other module. + case otherModule + } +} diff --git a/Sources/SwiftSourceKitPlugin/CodeCompletion/Completion+TypeRelation.swift b/Sources/SwiftSourceKitPlugin/CodeCompletion/Completion+TypeRelation.swift new file mode 100644 index 000000000..c66e4fe57 --- /dev/null +++ b/Sources/SwiftSourceKitPlugin/CodeCompletion/Completion+TypeRelation.swift @@ -0,0 +1,34 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +extension CompletionItem { + package enum TypeRelation { + /// The result does not have a type (e.g. keyword). + case notApplicable + + /// The type relation have not been calculated. + case unknown + + /// The relationship of the result's type to the expected type is not + /// invalid, not convertible, and not identical. + case unrelated + + /// The result's type is invalid at the expected position. + case invalid + + /// The result's type is convertible to the type of the expected. + case convertible + + /// The result's type is identical to the type of the expected. + case identical + } +} diff --git a/Sources/SwiftSourceKitPlugin/CodeCompletion/CompletionContext.swift b/Sources/SwiftSourceKitPlugin/CodeCompletion/CompletionContext.swift new file mode 100644 index 000000000..9e84c861c --- /dev/null +++ b/Sources/SwiftSourceKitPlugin/CodeCompletion/CompletionContext.swift @@ -0,0 +1,68 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import CompletionScoring + +/// General information about the code completion +struct CompletionContext { + let kind: CompletionContext.Kind + let memberAccessTypes: [String] + let baseExprScope: PopularityIndex.Scope? + + init(kind: CompletionContext.Kind, memberAccessTypes: [String], baseExprScope: PopularityIndex.Scope?) { + self.kind = kind + self.memberAccessTypes = memberAccessTypes + self.baseExprScope = baseExprScope + } + + enum Kind { + case none + case `import` + case unresolvedMember + case dotExpr + case stmtOrExpr + case postfixExprBeginning + case postfixExpr + case postfixExprParen + case keyPathExprObjC + case keyPathExprSwift + case typeDeclResultBeginning + case typeSimpleBeginning + case typeIdentifierWithDot + case typeIdentifierWithoutDot + case caseStmtKeyword + case caseStmtBeginning + case nominalMemberBeginning + case accessorBeginning + case attributeBegin + case attributeDeclParen + case poundAvailablePlatform + case callArg + case labeledTrailingClosure + case returnStmtExpr + case yieldStmtExpr + case forEachSequence + case afterPoundExpr + case afterPoundDirective + case platformConditon + case afterIfStmtElse + case genericRequirement + case precedenceGroup + case stmtLabel + case effectsSpecifier + case forEachPatternBeginning + case typeAttrBeginning + case optionalBinding + case forEachKeywordIn + case thenStmtExpr + } +} diff --git a/Sources/SwiftSourceKitPlugin/CodeCompletion/CompletionItem.swift b/Sources/SwiftSourceKitPlugin/CodeCompletion/CompletionItem.swift new file mode 100644 index 000000000..c2ffcd24f --- /dev/null +++ b/Sources/SwiftSourceKitPlugin/CodeCompletion/CompletionItem.swift @@ -0,0 +1,58 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import CompletionScoring + +/// A code completion item that should be returned to the client. +struct CompletionItem { + /// The label with which the item should be displayed in an IDE + let label: String + + /// The string that should be used to match against what the user type. + let filterText: String + + /// The module that defines the code completion item or `nil` if the item is not defined in a module, like a keyword. + let module: String? + + /// The type that the code completion item produces. + /// + /// Eg. the type of a variable or the return type of a function. `nil` for completions that don't have a type, like + /// keywords. + let typeName: String? + + /// The edits that should be made if the code completion is selected. + let textEdit: TextEdit + let kind: ItemKind + let isSystem: Bool + let textMatchScore: Double + let priorityBucket: PriorityBucket + let semanticScore: Double + let semanticClassification: SemanticClassification? + let id: Identifier + let hasDiagnostic: Bool + let groupID: Int? +} + +extension CompletionItem: CustomStringConvertible, CustomDebugStringConvertible { + var description: String { filterText } + var debugDescription: String { + """ + [\(kind)]\ + \(isSystem ? "[sys]" : "")\ + \(label);\ + \(typeName == nil ? "" : "type=\(typeName!)") \ + edit=\(textEdit); \ + pri=\(priorityBucket.rawValue); \ + index=\(id.index) + """ + } +} diff --git a/Sources/SwiftSourceKitPlugin/CodeCompletion/CompletionOptions.swift b/Sources/SwiftSourceKitPlugin/CodeCompletion/CompletionOptions.swift new file mode 100644 index 000000000..375776db8 --- /dev/null +++ b/Sources/SwiftSourceKitPlugin/CodeCompletion/CompletionOptions.swift @@ -0,0 +1,56 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +struct CompletionOptions { + private static let defaultMaxResults: Int = 200 + + /// Whether the label and type name in the code completion result should be annotated XML or plain text. + let annotateResults: Bool + + /// Whether object literals should be included in the code completion results. + let includeObjectLiterals: Bool + + /// Whether initializer calls should be included in top-level completions. + let addInitsToTopLevel: Bool + + /// If a function has defaulted arguments, whether we should produce two results (one without any defaulted arguments + /// and one with all defaulted arguments) or only one (with all defaulted arguments). + let addCallWithNoDefaultArgs: Bool + + /// Whether to include the semantic components computed by completion sorting in the results. + let includeSemanticComponents: Bool + + init( + annotateResults: Bool = false, + includeObjectLiterals: Bool = false, + addInitsToTopLevel: Bool = false, + addCallWithNoDefaultArgs: Bool = true, + includeSemanticComponents: Bool = false + ) { + self.annotateResults = annotateResults + self.includeObjectLiterals = includeObjectLiterals + self.addInitsToTopLevel = addInitsToTopLevel + self.addCallWithNoDefaultArgs = addCallWithNoDefaultArgs + self.includeSemanticComponents = includeSemanticComponents + } + + //// The maximum number of results we should return if the client requested `input` results. + static func maxResults(input: Int?) -> Int { + guard let maxResults = input, maxResults != 0 else { + return defaultMaxResults + } + if maxResults < 0 { + return Int.max // unlimited + } + return maxResults + } +} diff --git a/Sources/SwiftSourceKitPlugin/CodeCompletion/CompletionSorting.swift b/Sources/SwiftSourceKitPlugin/CodeCompletion/CompletionSorting.swift new file mode 100644 index 000000000..015dd1fdd --- /dev/null +++ b/Sources/SwiftSourceKitPlugin/CodeCompletion/CompletionSorting.swift @@ -0,0 +1,84 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import CompletionScoring +import Foundation + +struct CompletionSorting { + private let session: CompletionSession + // private let items: [ASTCompletionItem] + // private let filterCandidates: CandidateBatch + private let pattern: Pattern + + struct Match { + let score: CompletionScore + let index: Int + } + + init( + filterText: String, + in session: CompletionSession + ) { + self.session = session + self.pattern = Pattern(text: filterText) + } + + /// Invoke `callback` with the top `maxResults` results and their scores + /// The buffer passed to `callback` is only valid for the duration of `callback`. + /// Returns the return value of `callback`. + func withScoredAndFilter(maxResults: Int, _ callback: (UnsafeBufferPointer) -> T) -> T { + var matches: UnsafeMutableBufferPointer + defer { matches.deinitializeAllAndDeallocate() } + if pattern.text.isEmpty { + matches = .allocate(capacity: session.items.count) + for (index, item) in session.items.enumerated() { + matches.initialize( + index: index, + to: Match( + score: CompletionScore(textComponent: 1, semanticComponent: item.semanticScore(in: session)), + index: index + ) + ) + } + } else { + let candidateMatches = pattern.scoredMatches(in: session.filterCandidates, precision: .fast) + matches = .allocate(capacity: candidateMatches.count) + for (index, match) in candidateMatches.enumerated() { + let semanticScore = session.items[match.candidateIndex].semanticScore(in: session) + matches.initialize( + index: index, + to: Match( + score: CompletionScore(textComponent: match.textScore, semanticComponent: semanticScore), + index: match.candidateIndex + ) + ) + } + } + + "".withCString { emptyCString in + matches.selectTopKAndTruncate(min(maxResults, matches.count)) { + if $0.score != $1.score { + return $0.score > $1.score + } else { + // Secondary sort by name. This is important to do early since when the + // filter text is empty there will be many tied scores and we do not + // want non-deterministic results in top-level completions. + let lhs = session.items[$0.index].filterNameCString ?? emptyCString + let rhs = session.items[$1.index].filterNameCString ?? emptyCString + return strcmp(lhs, rhs) < 0 + } + } + } + + return callback(UnsafeBufferPointer(matches)) + } +} diff --git a/Sources/SwiftSourceKitPlugin/CodeCompletion/Connection.swift b/Sources/SwiftSourceKitPlugin/CodeCompletion/Connection.swift new file mode 100644 index 000000000..272cb3a6f --- /dev/null +++ b/Sources/SwiftSourceKitPlugin/CodeCompletion/Connection.swift @@ -0,0 +1,330 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import CompletionScoring +import Csourcekitd +import Foundation +import SKLogging +import SKUtilities +import SourceKitD + +extension PopularityIndex.Scope { + init(string name: String) { + if let dotIndex = name.firstIndex(of: ".") { + self.init( + container: String(name[name.index(after: dotIndex)...]), + module: String(name[..(_ stackSize: Int, execute block: @Sendable @escaping () -> T) -> T { + var result: T! = nil + nonisolated(unsafe) let workItem = DispatchWorkItem(block: { + result = block() + }) + let thread = Thread { + workItem.perform() + } + thread.stackSize = stackSize + thread.start() + workItem.wait() + return result! +} + +final class Connection { + enum Error: SourceKitPluginError, CustomStringConvertible { + case openingFileFailed(path: String) + /// An error that occurred inside swiftIDE while performing completion. + case swiftIDEError(String) + case cancelled + + var description: String { + switch self { + case .openingFileFailed(path: let path): + return "Could not open file '\(path)'" + case .swiftIDEError(let message): + return message + case .cancelled: + return "Request cancelled" + } + } + + func response(sourcekitd: any SourceKitD) -> SKDResponse { + switch self { + case .openingFileFailed, .swiftIDEError: + return SKDResponse(error: .failed, description: description, sourcekitd: sourcekitd) + case .cancelled: + return SKDResponse(error: .cancelled, description: "Request cancelled", sourcekitd: sourcekitd) + } + } + } + + fileprivate let logger = Logger(subsystem: "org.swift.sourcekit.service-plugin", category: "Connection") + + private let impl: swiftide_api_connection_t + let sourcekitd: SourceKitD + + /// The list of documents that are open in SourceKitD. The key is the file's path on disk or a pseudo-path that + /// uniquely identifies the document if it doesn't exist on disk. + private var documents: [String: Document] = [:] + + /// Information to construct `PopularityIndex`. + private var scopedPopularityDataPath: String? + private var popularModules: [String]? + private var notoriousModules: [String]? + + /// Cached data read from `scopedPopularityDataPath`. + private var _scopedPopularityData: LazyValue<[PopularityIndex.Scope: [String: Double]]?> = .uninitialized + + /// Cached index. + private var _popularityIndex: LazyValue = .uninitialized + + /// Deprecated. + /// NOTE: `PopularityTable` was replaced with `PopularityIndex`. We keep this + /// until all clients migrates to `PopularityIndex`. + private var onlyPopularCompletions: PopularityTable = .init() + + /// Recent completions that were accepted by the client. + private var recentCompletions: [String] = [] + + /// The stack size that should be used for all operations that end up invoking the type checker. + private let semanticStackSize = 8 << 20 // 8 MB. + + init(opaqueIDEInspectionInstance: UnsafeMutableRawPointer?, sourcekitd: SourceKitD) { + self.sourcekitd = sourcekitd + impl = sourcekitd.ideApi.connection_create_with_inspection_instance(opaqueIDEInspectionInstance) + } + + deinit { + sourcekitd.ideApi.connection_dispose(impl) + } + + func openDocument(path: String, contents: String, compilerArguments: [String]? = nil) { + if documents[path] != nil { + logger.error("Document at '\(path)' is already open") + } + documents[path] = Document(contents: contents, compilerArguments: compilerArguments) + sourcekitd.ideApi.set_file_contents(impl, path, contents) + } + + func editDocument(path: String, atUTF8Offset offset: Int, length: Int, newText: String) { + guard let document = documents[path] else { + logger.error("Document at '\(path)' is not open") + return + } + + document.lineTable.replace(utf8Offset: offset, length: length, with: newText) + + sourcekitd.ideApi.set_file_contents(impl, path, document.lineTable.content) + } + + func editDocument(path: String, edit: TextEdit) { + guard let document = documents[path] else { + logger.error("Document at '\(path)' is not open") + return + } + + document.lineTable.replace( + fromLine: edit.range.lowerBound.line - 1, + utf8Offset: edit.range.lowerBound.utf8Column - 1, + toLine: edit.range.upperBound.line - 1, + utf8Offset: edit.range.upperBound.utf8Column - 1, + with: edit.newText + ) + + sourcekitd.ideApi.set_file_contents(impl, path, document.lineTable.content) + } + + func closeDocument(path: String) { + if documents[path] == nil { + logger.error("Document at '\(path)' was not open") + } + documents[path] = nil + sourcekitd.ideApi.set_file_contents(impl, path, nil) + } + + func complete( + at loc: Location, + arguments reqArgs: [String]? = nil, + options: CompletionOptions, + handle: swiftide_api_request_handle_t? + ) throws -> CompletionSession { + let offset: Int = try { + if let lineTable = documents[loc.path]?.lineTable { + return lineTable.utf8OffsetOf(line: loc.line - 1, utf8Column: loc.utf8Column - 1) + } else { + // FIXME: move line:column translation into C++ impl. so that we can avoid reading the file an extra time here. + do { + logger.log("Received code completion request for file that wasn't open. Reading file contents from disk.") + let contents = try String(contentsOfFile: loc.path) + let lineTable = LineTable(contents) + return lineTable.utf8OffsetOf(line: loc.line - 1, utf8Column: loc.utf8Column - 1) + } catch { + throw Error.openingFileFailed(path: loc.path) + } + } + }() + + let arguments = reqArgs ?? documents[loc.path]?.compilerArguments ?? [] + + let result: swiftide_api_completion_response_t = withArrayOfCStrings(arguments) { cargs in + let req = sourcekitd.ideApi.completion_request_create(loc.path, UInt32(offset), cargs, UInt32(cargs.count)) + defer { sourcekitd.ideApi.completion_request_dispose(req) } + sourcekitd.ideApi.completion_request_set_annotate_result(req, options.annotateResults) + sourcekitd.ideApi.completion_request_set_include_objectliterals(req, options.includeObjectLiterals); + sourcekitd.ideApi.completion_request_set_add_inits_to_top_level(req, options.addInitsToTopLevel); + sourcekitd.ideApi.completion_request_set_add_call_with_no_default_args(req, options.addCallWithNoDefaultArgs); + + do { + let sourcekitd = self.sourcekitd + nonisolated(unsafe) let impl = impl + nonisolated(unsafe) let req = req + nonisolated(unsafe) let handle = handle + return withStackSize(semanticStackSize) { + sourcekitd.ideApi.complete_cancellable(impl, req, handle)! + } + } + } + + if sourcekitd.ideApi.completion_result_is_error(result) { + let errorDescription = String(cString: sourcekitd.ideApi.completion_result_get_error_description(result)!) + // Usually `CompletionSession` is responsible for disposing the result. + // Since we don't form a `CompletionSession`, dispose of the result manually. + sourcekitd.ideApi.completion_result_dispose(result) + throw Error.swiftIDEError(errorDescription) + } else if sourcekitd.ideApi.completion_result_is_cancelled(result) { + sourcekitd.ideApi.completion_result_dispose(result) + throw Error.cancelled + } + + return CompletionSession( + connection: self, + location: loc, + response: result, + options: options + ) + } + + func cancel(handle: RequestHandle) { + sourcekitd.ideApi.cancel_request(impl, handle.handle) + } + + func markCachedCompilerInstanceShouldBeInvalidated() { + sourcekitd.ideApi.connection_mark_cached_compiler_instance_should_be_invalidated(impl, nil) + } + + // MARK: 'PopularityIndex' APIs. + + func updatePopularityIndex( + scopedPopularityDataPath: String, + popularModules: [String], + notoriousModules: [String] + ) { + + // Clear the cache if necessary. + // We don't check the content of the path assuming it's not changed. + // For 'popular/notoriousModules', we expect around 200 elements. + if scopedPopularityDataPath != self.scopedPopularityDataPath { + self._popularityIndex = .uninitialized + self._scopedPopularityData = .uninitialized + } else if popularModules != self.popularModules || notoriousModules != self.notoriousModules { + self._popularityIndex = .uninitialized + } + + self.scopedPopularityDataPath = scopedPopularityDataPath + self.popularModules = popularModules + self.notoriousModules = notoriousModules + } + + private var scopedPopularityData: [PopularityIndex.Scope: [String: Double]]? { + _scopedPopularityData.cachedValueOrCompute { + guard let jsonPath = self.scopedPopularityDataPath else { + return nil + } + + // A codable representation of `PopularityIndex.symbolPopularity`. + struct ScopedSymbolPopularity: Codable { + let values: [String] + let scores: [Double] + + var table: [String: Double] { + var map = [String: Double]() + for (value, score) in zip(values, scores) { + map[value] = score + } + return map + } + } + + do { + let jsonURL = URL(fileURLWithPath: jsonPath) + let decoder = JSONDecoder() + let data = try Data(contentsOf: jsonURL) + let decoded = try decoder.decode([String: ScopedSymbolPopularity].self, from: data) + var result = [PopularityIndex.Scope: [String: Double]]() + for (rawScope, popularity) in decoded { + let scope = PopularityIndex.Scope(string: rawScope) + result[scope] = popularity.table + } + return result + } catch { + logger.error("Failed to read popularity data at '\(jsonPath)'") + return nil + } + } + } + + var popularityIndex: PopularityIndex? { + _popularityIndex.cachedValueOrCompute { + guard let scopedPopularityData, let popularModules, let notoriousModules else { + return nil + } + return PopularityIndex( + symbolReferencePercentages: scopedPopularityData, + notoriousSymbols: /*unused*/ [], + popularModules: popularModules, + notoriousModules: notoriousModules + ) + } + } + + // MARK: 'PopularityTable' APIs (DEPRECATED). + + func updatePopularAPI(popularityTable: PopularityTable) { + self.onlyPopularCompletions = popularityTable + } + + func updateRecentCompletions(_ recent: [String]) { + self.recentCompletions = recent + } + + var popularityTable: PopularityTable { + var result = onlyPopularCompletions + result.add(popularSymbols: recentCompletions) + return result + } +} + +private final class Document { + var lineTable: LineTable + var compilerArguments: [String]? = nil + + init(contents: String, compilerArguments: [String]? = nil) { + self.lineTable = LineTable(contents) + self.compilerArguments = compilerArguments + } +} diff --git a/Sources/SwiftSourceKitPlugin/CodeCompletion/Location.swift b/Sources/SwiftSourceKitPlugin/CodeCompletion/Location.swift new file mode 100644 index 000000000..26464ba35 --- /dev/null +++ b/Sources/SwiftSourceKitPlugin/CodeCompletion/Location.swift @@ -0,0 +1,28 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +struct Location: Equatable, CustomStringConvertible { + let path: String + let position: Position + + init(path: String, position: Position) { + self.path = path + self.position = position + } + + var line: Int { position.line } + var utf8Column: Int { position.utf8Column } + + var description: String { + "\(path):\(position)" + } +} diff --git a/Sources/SwiftSourceKitPlugin/CodeCompletion/Position.swift b/Sources/SwiftSourceKitPlugin/CodeCompletion/Position.swift new file mode 100644 index 000000000..44fbbb663 --- /dev/null +++ b/Sources/SwiftSourceKitPlugin/CodeCompletion/Position.swift @@ -0,0 +1,32 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +struct Position: Equatable, Comparable, CustomStringConvertible { + /// Line number within a document (one-based). + let line: Int + + /// UTF-9 code-unit offset from the start of a line (one-based). + let utf8Column: Int + + init(line: Int, utf8Column: Int) { + self.line = line + self.utf8Column = utf8Column + } + + static func < (lhs: Position, rhs: Position) -> Bool { + return (lhs.line, lhs.utf8Column) < (rhs.line, rhs.utf8Column) + } + + var description: String { + "\(line):\(utf8Column)" + } +} diff --git a/Sources/SwiftSourceKitPlugin/CodeCompletion/TextEdit.swift b/Sources/SwiftSourceKitPlugin/CodeCompletion/TextEdit.swift new file mode 100644 index 000000000..9bf0e4571 --- /dev/null +++ b/Sources/SwiftSourceKitPlugin/CodeCompletion/TextEdit.swift @@ -0,0 +1,25 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +struct TextEdit: CustomStringConvertible { + let range: Range + let newText: String + + init(range: Range, newText: String) { + self.range = range + self.newText = newText + } + + var description: String { + "{\(range.lowerBound)-\(range.upperBound)=\(newText)}" + } +} diff --git a/Sources/SwiftSourceKitPlugin/CodeCompletion/WithArrayOfCStrings.swift b/Sources/SwiftSourceKitPlugin/CodeCompletion/WithArrayOfCStrings.swift new file mode 100644 index 000000000..91e3c24ad --- /dev/null +++ b/Sources/SwiftSourceKitPlugin/CodeCompletion/WithArrayOfCStrings.swift @@ -0,0 +1,50 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +/// From SwiftPrivate.swift. + +/// Compute the prefix sum of `seq`. +private func scan( + _ seq: S, + _ initial: U, + _ combine: (U, S.Element) -> U +) -> [U] { + var result: [U] = [] + result.reserveCapacity(seq.underestimatedCount) + var runningResult = initial + for element in seq { + runningResult = combine(runningResult, element) + result.append(runningResult) + } + return result +} + +func withArrayOfCStrings( + _ args: [String], + _ body: ([UnsafeMutablePointer?]) -> R +) -> R { + let argsCounts = Array(args.map { $0.utf8.count + 1 }) + let argsOffsets = [0] + scan(argsCounts, 0, +) + let argsBufferSize = argsOffsets.last! + var argsBuffer: [UInt8] = [] + argsBuffer.reserveCapacity(argsBufferSize) + for arg in args { + argsBuffer.append(contentsOf: arg.utf8) + argsBuffer.append(0) + } + return argsBuffer.withUnsafeMutableBufferPointer { (argsBuffer) in + let ptr = UnsafeMutableRawPointer(argsBuffer.baseAddress!).bindMemory(to: CChar.self, capacity: argsBuffer.count) + var cStrings: [UnsafeMutablePointer?] = argsOffsets.map { ptr + $0 } + cStrings[cStrings.count - 1] = nil + return body(cStrings) + } +} diff --git a/Sources/SwiftSourceKitPlugin/CompletionProvider.swift b/Sources/SwiftSourceKitPlugin/CompletionProvider.swift new file mode 100644 index 000000000..cf2b457b8 --- /dev/null +++ b/Sources/SwiftSourceKitPlugin/CompletionProvider.swift @@ -0,0 +1,510 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import CompletionScoring +import Csourcekitd +import Foundation +import SKLogging +import SourceKitD +import SwiftSourceKitPluginCommon + +/// Parse a `[String: Popularity]` dictionary from an array of XPC dictionaries that looks as follows: +/// ``` +/// [ +/// { +/// "key.popularity.key": , +/// "key.popularity.value.int.billion": +/// }, +/// ... +/// ] +/// ``` +/// If a key occurs twice, we use the later value. +/// Returns `nil` if parsing failed because one of he entries didn't contain a key or value. +private func parsePopularityDict(_ data: SKDRequestArrayReader) -> [String: Popularity]? { + var result: [String: Popularity] = [:] + let iteratedAllEntries = data.forEach { (_, entry) -> Bool in + // We can't deserialize double values in SourceKit requests at the moment. + // We transfer the double value as an integer with 9 significant digits by multiplying it by 1 billion first. + guard let key: String = entry[entry.sourcekitd.keys.popularityKey], + let value: Int = entry[entry.sourcekitd.keys.popularityValueIntBillion] + else { + return false + } + result[key] = Popularity(scoreComponent: Double(value) / 1_000_000_000) + return true + } + if !iteratedAllEntries { + return nil + } + return result +} + +extension PopularityTable { + /// Create a PopularityTable from a serialized XPC form that looks as follows: + /// ``` + /// { + /// "key.symbol_popularity": [ ], + /// "key.module_popularity": [ ], + /// } + /// ``` + /// Returns `nil` if the dictionary didn't match the expected format. + init?(_ dict: SKDRequestDictionaryReader) { + let keys = dict.sourcekitd.keys + guard let symbolPopularityData: SKDRequestArrayReader = dict[keys.symbolPopularity], + let symbolPopularity = parsePopularityDict(symbolPopularityData), + let modulePopularityData: SKDRequestArrayReader = dict[keys.modulePopularity], + let modulePopularity = parsePopularityDict(modulePopularityData) + else { + return nil + } + self.init(symbolPopularity: symbolPopularity, modulePopularity: modulePopularity) + } +} + +actor CompletionProvider { + enum InvalidRequest: SourceKitPluginError { + case missingKey(String) + + func response(sourcekitd: SourceKitD) -> SKDResponse { + switch self { + case .missingKey(let key): + return SKDResponse(error: .invalid, description: "missing required key '\(key)'", sourcekitd: sourcekitd) + } + } + } + + private let logger = Logger(subsystem: "org.swift.sourcekit.service-plugin", category: "CompletionProvider") + + private let connection: Connection + + /// The XPC custom buffer kind for `CompletionResultsArray` + private let completionResultsBufferKind: UInt64 + + /// The code completion session that's currently open. + private var currentSession: CompletionSession? = nil + + init( + completionResultsBufferKind: UInt64, + opaqueIDEInspectionInstance: OpaqueIDEInspectionInstance? = nil, + sourcekitd: SourceKitD + ) { + self.connection = Connection( + opaqueIDEInspectionInstance: opaqueIDEInspectionInstance?.value, + sourcekitd: sourcekitd + ) + self.completionResultsBufferKind = completionResultsBufferKind + } + + nonisolated func cancel(handle: RequestHandle) { + Task { + await self.cancelImpl(handle: handle) + } + } + + private func cancelImpl(handle: RequestHandle) { + connection.cancel(handle: handle) + } + + func handleDocumentOpen(_ request: SKDRequestDictionaryReader) { + let keys = request.sourcekitd.keys + guard let path: String = request[keys.name] else { + self.logger.error("error: dropping request editor.open: missing 'key.name'") + return + } + let content: String + if let text: String = request[keys.sourceText] { + content = text + } else if let file: String = request[keys.sourceFile] { + logger.info("Document open request missing source text. Reading contents of '\(file)' from disk.") + do { + content = try String(contentsOfFile: file) + } catch { + self.logger.error("error: dropping request editor.open: failed to read \(file): \(String(describing: error))") + return + } + } else { + self.logger.error("error: dropping request editor.open: missing 'key.sourcetext'") + return + } + + self.connection.openDocument( + path: path, + contents: content, + compilerArguments: request[keys.compilerArgs]?.asStringArray + ) + } + + func handleDocumentEdit(_ request: SKDRequestDictionaryReader) { + let keys = request.sourcekitd.keys + guard let path: String = request[keys.name] else { + self.logger.error("error: dropping request editor.replacetext: missing 'key.name'") + return + } + guard let offset: Int = request[keys.offset] else { + self.logger.error("error: dropping request editor.replacetext: missing 'key.offset'") + return + } + guard let length: Int = request[keys.length] else { + self.logger.error("error: dropping request editor.replacetext: missing 'key.length'") + return + } + guard let text: String = request[keys.sourceText] else { + self.logger.error("error: dropping request editor.replacetext: missing 'key.sourcetext'") + return + } + + self.connection.editDocument(path: path, atUTF8Offset: offset, length: length, newText: text) + } + + func handleDocumentClose(_ dict: SKDRequestDictionaryReader) { + guard let path: String = dict[dict.sourcekitd.keys.name] else { + self.logger.error("error: dropping request editor.close: missing 'key.name'") + return + } + self.connection.closeDocument(path: path) + } + + func handleCompleteOpen( + _ request: SKDRequestDictionaryReader, + handle: RequestHandle? + ) throws -> SKDResponseDictionaryBuilder { + let sourcekitd = request.sourcekitd + let keys = sourcekitd.keys + let location = try self.requestLocation(request) + + if self.currentSession != nil { + logger.error("Opening a code completion session while previous is still open. Implicitly closing old session.") + self.currentSession = nil + } + + let options: SKDRequestDictionaryReader? = request[keys.codeCompleteOptions] + let annotate = (options?[keys.annotatedDescription] as Int?) == 1 + let includeObjectLiterals = (options?[keys.includeObjectLiterals] as Int?) == 1 + let addInitsToTopLevel = (options?[keys.addInitsToTopLevel] as Int?) == 1 + let addCallWithNoDefaultArgs = (options?[keys.addCallWithNoDefaultArgs] as Int? == 1) + let includeSemanticComponents = (options?[keys.includeSemanticComponents] as Int?) == 1 + + if let recentCompletions: [String] = options?[keys.recentCompletions]?.asStringArray { + self.connection.updateRecentCompletions(recentCompletions) + } + + let session = try self.connection.complete( + at: location, + arguments: request[keys.compilerArgs]?.asStringArray, + options: CompletionOptions( + annotateResults: annotate, + includeObjectLiterals: includeObjectLiterals, + addInitsToTopLevel: addInitsToTopLevel, + addCallWithNoDefaultArgs: addCallWithNoDefaultArgs, + includeSemanticComponents: includeSemanticComponents + ), + handle: handle?.handle + ) + + self.currentSession = session + + return completionsResponse(session: session, options: options, sourcekitd: sourcekitd) + } + + func handleCompleteUpdate(_ request: SKDRequestDictionaryReader) throws -> SKDResponseDictionaryBuilder { + let sourcekitd = request.sourcekitd + let location = try self.requestLocation(request) + + let options: SKDRequestDictionaryReader? = request[sourcekitd.keys.codeCompleteOptions] + + guard let session = self.currentSession, session.location == location else { + throw GenericPluginError(description: "no matching session for \(location)") + } + + return completionsResponse(session: session, options: options, sourcekitd: sourcekitd) + } + + func handleCompleteClose(_ dict: SKDRequestDictionaryReader) throws -> SKDResponseDictionaryBuilder { + let sourcekitd = dict.sourcekitd + + let location = try self.requestLocation(dict) + + guard let session = self.currentSession, session.location == location else { + throw GenericPluginError(description: "no matching session for \(location)") + } + + self.currentSession = nil + return sourcekitd.responseDictionary([:]) + } + + func handleExtendedCompletionRequest(_ request: SKDRequestDictionaryReader) throws -> ExtendedCompletionInfo { + let sourcekitd = request.sourcekitd + let keys = sourcekitd.keys + + guard let opaqueID: Int64 = request[keys.identifier] else { + throw InvalidRequest.missingKey("key.identifier") + } + + guard let session = self.currentSession else { + throw GenericPluginError(description: "no matching session for request \(request)") + } + + let id = CompletionItem.Identifier(opaqueValue: opaqueID) + guard let info = session.extendedCompletionInfo(for: id) else { + throw GenericPluginError(description: "unknown completion \(opaqueID) for session at \(session.location)") + } + + return info + } + + func handleCompletionDocumentation(_ request: SKDRequestDictionaryReader) throws -> SKDResponseDictionaryBuilder { + let info = try handleExtendedCompletionRequest(request) + + return request.sourcekitd.responseDictionary([ + request.sourcekitd.keys.docBrief: info.briefDocumentation, + request.sourcekitd.keys.associatedUSRs: info.associatedUSRs as [SKDResponseValue]?, + ]) + } + + func handleCompletionDiagnostic(_ dict: SKDRequestDictionaryReader) throws -> SKDResponseDictionaryBuilder { + let info = try handleExtendedCompletionRequest(dict) + let sourcekitd = dict.sourcekitd + + let severity: sourcekitd_api_uid_t? = + switch info.diagnostic?.severity { + case .note: sourcekitd.values.diagNote + case .remark: sourcekitd.values.diagRemark + case .warning: sourcekitd.values.diagWarning + case .error: sourcekitd.values.diagError + default: nil + } + return sourcekitd.responseDictionary([ + sourcekitd.keys.severity: severity, + sourcekitd.keys.description: info.diagnostic?.description, + ]) + } + + func handleDependencyUpdated() { + connection.markCachedCompilerInstanceShouldBeInvalidated() + } + + func handleSetPopularAPI(_ dict: SKDRequestDictionaryReader) -> SKDResponseDictionaryBuilder { + let sourcekitd = dict.sourcekitd + let keys = sourcekitd.keys + + let didUseScoreComponents: Bool + + // Try 'PopularityIndex' scheme first, then fall back to `PopularityTable` + // scheme. + if let scopedPopularityDataPath: String = dict[keys.scopedPopularityTablePath] { + // NOTE: Currently, the client sends setpopularapi before every + // 'complete.open' because sourcekit might have crashed before it. + // 'scoped_popularity_table_path' and its content typically do not + // change in the 'Connection' lifetime, but 'popular_modules'/'notorious_modules' + // might. We cache the populated table, and use it as long as the these + // values are the same as the previous request. + self.connection.updatePopularityIndex( + scopedPopularityDataPath: scopedPopularityDataPath, + popularModules: dict[keys.popularModules]?.asStringArray ?? [], + notoriousModules: dict[keys.notoriousModules]?.asStringArray ?? [] + ) + didUseScoreComponents = true + } else if let popularityTable = PopularityTable(dict) { + self.connection.updatePopularAPI(popularityTable: popularityTable) + didUseScoreComponents = true + } else { + let popular: [String] = dict[keys.popular]?.asStringArray ?? [] + let unpopular: [String] = dict[keys.unpopular]?.asStringArray ?? [] + let popularityTable = PopularityTable(popularSymbols: popular, recentSymbols: [], notoriousSymbols: unpopular) + self.connection.updatePopularAPI(popularityTable: popularityTable) + didUseScoreComponents = false + } + return sourcekitd.responseDictionary([ + keys.useNewAPI: 1, // Make it possible to detect this was handled by the plugin. + keys.usedScoreComponents: didUseScoreComponents ? 1 : 0, + ]) + } + + private func requestLocation(_ dict: SKDRequestDictionaryReader) throws -> Location { + let keys = dict.sourcekitd.keys + guard let path: String = dict[keys.sourceFile] else { + throw InvalidRequest.missingKey("key.sourcefile") + } + guard let line: Int = dict[keys.line] else { + throw InvalidRequest.missingKey("key.line") + } + guard let column: Int = dict[keys.column] else { + throw InvalidRequest.missingKey("key.column") + } + return Location(path: path, position: Position(line: line, utf8Column: column)) + } + + private func populateCompletionsXPC( + _ completions: [CompletionItem], + in session: CompletionSession, + into resp: inout SKDResponseDictionaryBuilder, + sourcekitd: SourceKitD + ) { + let keys = sourcekitd.keys + + let options = session.options + if options.annotateResults { + resp.set(keys.annotatedTypeName, to: true) + } + + let results = + completions.map { item in + sourcekitd.responseDictionary([ + keys.kind: sourcekitd_api_uid_t(item.kind, sourcekitd: sourcekitd), + keys.identifier: item.id.opaqueValue, + keys.name: item.filterText, + keys.description: item.label, + keys.sourceText: item.textEdit.newText, + keys.isSystem: item.isSystem ? 1 : 0, + keys.numBytesToErase: item.numBytesToErase(from: session.location.position), + keys.typeName: item.typeName ?? "", // FIXME: make it optional? + keys.textMatchScore: item.textMatchScore, + keys.semanticScore: item.semanticScore, + keys.semanticScoreComponents: options.includeSemanticComponents ? nil : item.semanticClassification?.asBase64, + keys.priorityBucket: item.priorityBucket.rawValue, + keys.hasDiagnostic: item.hasDiagnostic ? 1 : 0, + keys.groupId: item.groupID, + ]) + } as [SKDResponseValue] + resp.set(sourcekitd.keys.results, to: results) + } + + private func populateCompletions( + _ completions: [CompletionItem], + in session: CompletionSession, + into resp: inout SKDResponseDictionaryBuilder, + includeSemanticComponents: Bool, + sourcekitd: SourceKitD + ) { + let keys = sourcekitd.keys + + let options = session.options + if options.annotateResults { + resp.set(keys.annotatedTypeName, to: true) + } + + var builder = CompletionResultsArrayBuilder( + bufferKind: self.completionResultsBufferKind, + numResults: completions.count, + session: session + ) + for item in completions { + builder.add(item, includeSemanticComponents: includeSemanticComponents, sourcekitd: sourcekitd) + } + + let bytes = builder.bytes() + bytes.withUnsafeBytes { buffer in + resp.set(keys.results, toCustomBuffer: buffer) + } + } + + private func completionsResponse( + session: CompletionSession, + options: SKDRequestDictionaryReader?, + sourcekitd: SourceKitD + ) -> SKDResponseDictionaryBuilder { + let keys = sourcekitd.keys + var response = sourcekitd.responseDictionary([ + keys.unfilteredResultCount: session.totalCount, + keys.memberAccessTypes: session.memberAccessTypes as [SKDResponseValue], + ]) + + let filterText = options?[keys.filterText] ?? "" + let maxResults = CompletionOptions.maxResults(input: options?[keys.maxResults]) + let includeSemanticComponents = (options?[keys.includeSemanticComponents] as Int?) == 1 + + let completions = session.completions(matchingFilterText: filterText, maxResults: maxResults) + + if let useXPC: Int = options?[keys.useXPCSerialization], useXPC != 0 { + self.populateCompletionsXPC(completions, in: session, into: &response, sourcekitd: sourcekitd) + } else { + self.populateCompletions( + completions, + in: session, + into: &response, + includeSemanticComponents: includeSemanticComponents, + sourcekitd: sourcekitd + ) + } + return response + } +} + +extension sourcekitd_api_uid_t { + init(_ itemKind: CompletionItem.ItemKind, isRef: Bool = false, sourcekitd: SourceKitD) { + switch itemKind { + case .module: + self = isRef ? sourcekitd.values.refModule : sourcekitd.values.declModule + case .class: + self = isRef ? sourcekitd.values.refClass : sourcekitd.values.declClass + case .actor: + self = isRef ? sourcekitd.values.refActor : sourcekitd.values.declActor + case .struct: + self = isRef ? sourcekitd.values.refStruct : sourcekitd.values.declStruct + case .enum: + self = isRef ? sourcekitd.values.refEnum : sourcekitd.values.declEnum + case .enumElement: + self = isRef ? sourcekitd.values.refEnumElement : sourcekitd.values.declEnumElement + case .protocol: + self = isRef ? sourcekitd.values.refProtocol : sourcekitd.values.declProtocol + case .associatedType: + self = isRef ? sourcekitd.values.refAssociatedType : sourcekitd.values.declAssociatedType + case .typeAlias: + self = isRef ? sourcekitd.values.refTypeAlias : sourcekitd.values.declTypeAlias + case .genericTypeParam: + self = isRef ? sourcekitd.values.refGenericTypeParam : sourcekitd.values.declGenericTypeParam + case .constructor: + self = isRef ? sourcekitd.values.refConstructor : sourcekitd.values.declConstructor + case .destructor: + self = isRef ? sourcekitd.values.refDestructor : sourcekitd.values.declDestructor + case .subscript: + self = isRef ? sourcekitd.values.refSubscript : sourcekitd.values.declSubscript + case .staticMethod: + self = isRef ? sourcekitd.values.refMethodStatic : sourcekitd.values.declMethodStatic + case .instanceMethod: + self = isRef ? sourcekitd.values.refMethodInstance : sourcekitd.values.declMethodInstance + case .prefixOperatorFunction: + self = isRef ? sourcekitd.values.refFunctionPrefixOperator : sourcekitd.values.declFunctionPrefixOperator + case .postfixOperatorFunction: + self = isRef ? sourcekitd.values.refFunctionPostfixOperator : sourcekitd.values.declFunctionPostfixOperator + case .infixOperatorFunction: + self = isRef ? sourcekitd.values.refFunctionInfixOperator : sourcekitd.values.declFunctionInfixOperator + case .freeFunction: + self = isRef ? sourcekitd.values.refFunctionFree : sourcekitd.values.declFunctionFree + case .staticVar: + self = isRef ? sourcekitd.values.refVarStatic : sourcekitd.values.declVarStatic + case .instanceVar: + self = isRef ? sourcekitd.values.refVarInstance : sourcekitd.values.declVarInstance + case .localVar: + self = isRef ? sourcekitd.values.refVarLocal : sourcekitd.values.declVarLocal + case .globalVar: + self = isRef ? sourcekitd.values.refVarGlobal : sourcekitd.values.declVarGlobal + case .precedenceGroup: + self = isRef ? sourcekitd.values.refPrecedenceGroup : sourcekitd.values.declPrecedenceGroup + case .macro: + self = isRef ? sourcekitd.values.refMacro : sourcekitd.values.declMacro + case .keyword: + self = sourcekitd.values.completionKindKeyword + case .operator: + // FIXME: special operator ? + self = sourcekitd.values.completionKindPattern + case .literal: + // FIXME: special literal ? + self = sourcekitd.values.completionKindKeyword + case .pattern: + self = sourcekitd.values.completionKindPattern + case .unknown: + // FIXME: special unknown ? + self = sourcekitd.values.completionKindKeyword + } + } +} diff --git a/Sources/SwiftSourceKitPlugin/CompletionResultsArrayBuilder.swift b/Sources/SwiftSourceKitPlugin/CompletionResultsArrayBuilder.swift new file mode 100644 index 000000000..8bfe7bcee --- /dev/null +++ b/Sources/SwiftSourceKitPlugin/CompletionResultsArrayBuilder.swift @@ -0,0 +1,116 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import CompletionScoring +import Csourcekitd +import Foundation +import SourceKitD +import SwiftSourceKitPluginCommon + +struct CompletionResultsArrayBuilder { + private let bufferKind: UInt64 + private var results: [CompletionResult] = [] + private var stringTable: [String: Int] = [:] + private var nextString: Int = 0 + private let startLoc: Position + + init(bufferKind: UInt64, numResults: Int, session: CompletionSession) { + self.bufferKind = bufferKind + self.results.reserveCapacity(numResults) + self.stringTable.reserveCapacity(numResults * 3) + self.startLoc = session.location.position + } + + private mutating func addString(_ str: String) -> CompletionResult.StringEntry { + if let value = stringTable[str] { + return CompletionResult.StringEntry(start: UInt32(value)) + } else { + let value = nextString + precondition(value < Int(UInt32.max)) + nextString += str.utf8.count + 1 + + stringTable[str] = value + return CompletionResult.StringEntry(start: UInt32(value)) + } + } + + private mutating func addString(_ str: String?) -> CompletionResult.StringEntry? { + guard let str else { + return nil + } + return addString(str) as CompletionResult.StringEntry + } + + mutating func add(_ item: CompletionItem, includeSemanticComponents: Bool, sourcekitd: SourceKitD) { + let result = CompletionResult( + kind: sourcekitd_api_uid_t(item.kind, sourcekitd: sourcekitd), + identifier: item.id.opaqueValue, + name: addString(item.filterText), + description: addString(item.label), + sourceText: addString(item.textEdit.newText), + module: addString(item.module), + typename: addString(item.typeName ?? ""), + textMatchScore: item.textMatchScore, + semanticScore: item.semanticScore, + semanticScoreComponents: includeSemanticComponents ? addString(item.semanticClassification?.asBase64) : nil, + priorityBucket: Int32(item.priorityBucket.rawValue), + isSystem: item.isSystem, + numBytesToErase: item.numBytesToErase(from: startLoc), + hasDiagnostic: item.hasDiagnostic, + groupID: Int64(item.groupID ?? 0) + ) + results.append(result) + } + + func bytes() -> [UInt8] { + let capacity = + MemoryLayout.size // kind + + MemoryLayout.size // numResults + + results.count * MemoryLayout.stride + nextString + + return Array(unsafeUninitializedCapacity: capacity) { + (bytes: inout UnsafeMutableBufferPointer, size: inout Int) in + size = capacity + var cursor = UnsafeMutableRawBufferPointer(bytes) + cursor.storeBytes(of: self.bufferKind, toByteOffset: 0, as: UInt64.self) + cursor = UnsafeMutableRawBufferPointer(rebasing: cursor[MemoryLayout.size...]) + cursor.storeBytes(of: self.results.count, toByteOffset: 0, as: Int.self) + cursor = UnsafeMutableRawBufferPointer(rebasing: cursor[MemoryLayout.size...]) + self.results.withUnsafeBytes { raw in + cursor.copyMemory(from: raw) + cursor = UnsafeMutableRawBufferPointer(rebasing: cursor[raw.count...]) + } + for (str, startOffset) in stringTable { + let slice = UnsafeMutableRawBufferPointer(rebasing: cursor[startOffset...]) + str.utf8CString.withUnsafeBytes { raw in + slice.copyMemory(from: raw) + } + } + } + } +} + +extension CompletionItem { + func numBytesToErase(from: Position) -> Int { + guard textEdit.range.lowerBound.line == from.line else { + assertionFailure("unsupported multi-line completion edit start \(from) vs \(textEdit)") + return 0 + } + return from.utf8Column - textEdit.range.lowerBound.utf8Column + } +} + +extension SemanticClassification { + var asBase64: String { + return Data(self.byteRepresentation()).base64EncodedString() + } +} diff --git a/Sources/SwiftSourceKitPlugin/Plugin.swift b/Sources/SwiftSourceKitPlugin/Plugin.swift new file mode 100644 index 000000000..4f65383f6 --- /dev/null +++ b/Sources/SwiftSourceKitPlugin/Plugin.swift @@ -0,0 +1,200 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation +import SourceKitD +import SwiftExtensions +import SwiftSourceKitPluginCommon + +#if compiler(>=6) +public import Csourcekitd +#else +import Csourcekitd +#endif + +private func useNewAPI(for dict: SKDRequestDictionaryReader) -> Bool { + guard let opts: SKDRequestDictionaryReader = dict[dict.sourcekitd.keys.codeCompleteOptions], + opts[dict.sourcekitd.keys.useNewAPI] == 1 + else { + return false + } + return true +} + +final class RequestHandler: Sendable { + enum HandleRequestResult { + /// `handleRequest` will call `receiver`. + case requestHandled + + /// `handleRequest` will not call `receiver` and a request response should be produced by sourcekitd (not the plugin). + case handleInSourceKitD + } + + let requestHandlingQueue = AsyncQueue() + let sourcekitd: SourceKitD + let completionProvider: CompletionProvider + + init(params: sourcekitd_api_plugin_initialize_params_t, completionResultsBufferKind: UInt64, sourcekitd: SourceKitD) { + let ideInspectionInstance = sourcekitd.servicePluginApi.plugin_initialize_get_swift_ide_inspection_instance(params) + + self.sourcekitd = sourcekitd + self.completionProvider = CompletionProvider( + completionResultsBufferKind: completionResultsBufferKind, + opaqueIDEInspectionInstance: OpaqueIDEInspectionInstance(ideInspectionInstance), + sourcekitd: sourcekitd + ) + } + + func handleRequest( + _ dict: SKDRequestDictionaryReader, + handle: RequestHandle?, + receiver: @Sendable @escaping (SKDResponse) -> Void + ) -> HandleRequestResult { + func produceResult( + body: @escaping @Sendable () async throws -> SKDResponseDictionaryBuilder + ) -> HandleRequestResult { + requestHandlingQueue.async { + do { + receiver(try await body().response) + } catch { + receiver(SKDResponse.from(error: error, sourcekitd: self.sourcekitd)) + } + } + return .requestHandled + } + + func sourcekitdProducesResult(body: @escaping @Sendable () async -> ()) -> HandleRequestResult { + requestHandlingQueue.async { + await body() + } + return .handleInSourceKitD + } + + switch dict[sourcekitd.keys.request] as sourcekitd_api_uid_t? { + case sourcekitd.requests.editorOpen: + return sourcekitdProducesResult { + await self.completionProvider.handleDocumentOpen(dict) + } + case sourcekitd.requests.editorReplaceText: + return sourcekitdProducesResult { + await self.completionProvider.handleDocumentEdit(dict) + } + case sourcekitd.requests.editorClose: + return sourcekitdProducesResult { + await self.completionProvider.handleDocumentClose(dict) + } + + case sourcekitd.requests.codeCompleteOpen: + guard useNewAPI(for: dict) else { + return .handleInSourceKitD + } + return produceResult { + try await self.completionProvider.handleCompleteOpen(dict, handle: handle) + } + case sourcekitd.requests.codeCompleteUpdate: + guard useNewAPI(for: dict) else { + return .handleInSourceKitD + } + return produceResult { + try await self.completionProvider.handleCompleteUpdate(dict) + } + case sourcekitd.requests.codeCompleteClose: + guard useNewAPI(for: dict) else { + return .handleInSourceKitD + } + return produceResult { + try await self.completionProvider.handleCompleteClose(dict) + } + case sourcekitd.requests.codeCompleteDocumentation: + return produceResult { + try await self.completionProvider.handleCompletionDocumentation(dict) + } + case sourcekitd.requests.codeCompleteDiagnostic: + return produceResult { + try await self.completionProvider.handleCompletionDiagnostic(dict) + } + case sourcekitd.requests.codeCompleteSetPopularAPI: + guard useNewAPI(for: dict) else { + return .handleInSourceKitD + } + return produceResult { + await self.completionProvider.handleSetPopularAPI(dict) + } + case sourcekitd.requests.dependencyUpdated: + return sourcekitdProducesResult { + await self.completionProvider.handleDependencyUpdated() + } + default: + return .handleInSourceKitD + } + } + + func cancel(_ handle: RequestHandle) { + self.completionProvider.cancel(handle: handle) + } +} + +@_cdecl("sourcekitd_plugin_initialize") +public func sourcekitd_plugin_initialize(_ params: sourcekitd_api_plugin_initialize_params_t) { + let sourcekitd = DynamicallyLoadedSourceKitD.relativeToPlugin + + let completionResultsBufferKind = sourcekitd.pluginApi.plugin_initialize_custom_buffer_start(params) + let isClientOnly = sourcekitd.pluginApi.plugin_initialize_is_client_only(params) + + let uidFromCString = sourcekitd.pluginApi.plugin_initialize_uid_get_from_cstr(params) + let uidGetCString = sourcekitd.pluginApi.plugin_initialize_uid_get_string_ptr(params) + + // Depending on linking and loading configuration, we may need to chain the global UID handlers back to the UID + // handlers in the caller. The extra hop should not matter, since we cache the results. + if unsafeBitCast(uidFromCString, to: UnsafeRawPointer.self) + != unsafeBitCast(sourcekitd.api.uid_get_from_cstr, to: UnsafeRawPointer.self) + { + sourcekitd.api.set_uid_handlers(uidFromCString, uidGetCString) + } + + sourcekitd.pluginApi.plugin_initialize_register_custom_buffer( + params, + completionResultsBufferKind, + CompletionResultsArray.arrayFuncs.rawValue + ) + + if isClientOnly { + return + } + + let requestHandler = RequestHandler( + params: params, + completionResultsBufferKind: completionResultsBufferKind, + sourcekitd: sourcekitd + ) + + sourcekitd.servicePluginApi.plugin_initialize_register_cancellation_handler(params) { handle in + if let handle = RequestHandle(handle) { + requestHandler.cancel(handle) + } + } + + sourcekitd.servicePluginApi.plugin_initialize_register_cancellable_request_handler(params) { + (request, handle, receiver) -> Bool in + guard let receiver, let request, let dict = SKDRequestDictionaryReader(request, sourcekitd: sourcekitd) else { + return false + } + let handle = RequestHandle(handle) + + let handledRequest = requestHandler.handleRequest(dict, handle: handle) { receiver($0.underlyingValueRetained()) } + + switch handledRequest { + case .requestHandled: return true + case .handleInSourceKitD: return false + } + } +} diff --git a/Sources/SwiftSourceKitPlugin/SKDRequestArrayReader.swift b/Sources/SwiftSourceKitPlugin/SKDRequestArrayReader.swift new file mode 100644 index 000000000..4ab7eb639 --- /dev/null +++ b/Sources/SwiftSourceKitPlugin/SKDRequestArrayReader.swift @@ -0,0 +1,69 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Csourcekitd +import SourceKitD + +/// Provide getters to get values for a sourcekitd request array. +/// +/// This is not part of the `SourceKitD` module because it uses `SourceKitD.servicePluginAPI` which must not be accessed +/// outside of the service plugin. +final class SKDRequestArrayReader: Sendable { + nonisolated(unsafe) let array: sourcekitd_api_object_t + private let sourcekitd: SourceKitD + + /// Creates an `SKDRequestArray` that essentially provides a view into the given opaque `sourcekitd_api_object_t`. + init(_ array: sourcekitd_api_object_t, sourcekitd: SourceKitD) { + self.array = array + self.sourcekitd = sourcekitd + _ = sourcekitd.api.request_retain(array) + } + + deinit { + _ = sourcekitd.api.request_release(array) + } + + var count: Int { return sourcekitd.servicePluginApi.request_array_get_count(array) } + + /// If the `applier` returns `false`, iteration terminates. + @discardableResult + func forEach(_ applier: (Int, SKDRequestDictionaryReader) throws -> Bool) rethrows -> Bool { + for i in 0.. String? { + if let cstr = sourcekitd.servicePluginApi.request_array_get_string(array, index) { + return String(cString: cstr) + } + return nil + } + + var asStringArray: [String] { + var result: [String] = [] + for i in 0..( + _ key: sourcekitd_api_uid_t, + _ variantType: sourcekitd_api_variant_type_t, + _ retrievalFunction: (sourcekitd_api_object_t) -> T? + ) -> T? { + guard let value = sourcekitd.servicePluginApi.request_dictionary_get_value(sourcekitd_api_object_t(dict), key) + else { + return nil + } + if sourcekitd.servicePluginApi.request_get_type(value) == variantType { + return retrievalFunction(value) + } else { + return nil + } + } + + subscript(key: sourcekitd_api_uid_t) -> String? { + return sourcekitd.servicePluginApi.request_dictionary_get_string(sourcekitd_api_object_t(dict), key).map( + String.init(cString:) + ) + } + + subscript(key: sourcekitd_api_uid_t) -> Int64? { + return getVariant(key, SOURCEKITD_API_VARIANT_TYPE_INT64, sourcekitd.servicePluginApi.request_int64_get_value) + } + + subscript(key: sourcekitd_api_uid_t) -> Int? { + guard let value: Int64 = self[key] else { + return nil + } + return Int(value) + } + + subscript(key: sourcekitd_api_uid_t) -> Bool? { + return getVariant(key, SOURCEKITD_API_VARIANT_TYPE_BOOL, sourcekitd.servicePluginApi.request_bool_get_value) + } + + subscript(key: sourcekitd_api_uid_t) -> sourcekitd_api_uid_t? { + return sourcekitd.servicePluginApi.request_dictionary_get_uid(sourcekitd_api_object_t(dict), key) + } + + subscript(key: sourcekitd_api_uid_t) -> SKDRequestArrayReader? { + return getVariant(key, SOURCEKITD_API_VARIANT_TYPE_ARRAY) { + SKDRequestArrayReader($0, sourcekitd: sourcekitd) + } + } + + subscript(key: sourcekitd_api_uid_t) -> SKDRequestDictionaryReader? { + return getVariant(key, SOURCEKITD_API_VARIANT_TYPE_DICTIONARY) { + SKDRequestDictionaryReader($0, sourcekitd: sourcekitd) + } + } +} diff --git a/Sources/SwiftSourceKitPlugin/SKDResponse.swift b/Sources/SwiftSourceKitPlugin/SKDResponse.swift new file mode 100644 index 000000000..661faf76b --- /dev/null +++ b/Sources/SwiftSourceKitPlugin/SKDResponse.swift @@ -0,0 +1,70 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Csourcekitd +import Darwin +import SourceKitD + +final class SKDResponse: CustomStringConvertible, Sendable { + enum ErrorKind { + case connectionInterrupted + case invalid + case failed + case cancelled + + fileprivate var underlyingError: sourcekitd_api_error_t { + switch self { + case .connectionInterrupted: return SOURCEKITD_API_ERROR_CONNECTION_INTERRUPTED + case .invalid: return SOURCEKITD_API_ERROR_REQUEST_INVALID + case .failed: return SOURCEKITD_API_ERROR_REQUEST_FAILED + case .cancelled: return SOURCEKITD_API_ERROR_REQUEST_CANCELLED + } + } + } + + nonisolated(unsafe) let value: sourcekitd_api_response_t + let sourcekitd: SourceKitD + + init(takingUnderlyingResponse value: sourcekitd_api_response_t, sourcekitd: SourceKitD) { + self.value = value + self.sourcekitd = sourcekitd + } + + convenience init(error errorKind: ErrorKind, description: String, sourcekitd: SourceKitD) { + let resp = sourcekitd.servicePluginApi.response_error_create(errorKind.underlyingError, description)! + self.init(takingUnderlyingResponse: resp, sourcekitd: sourcekitd) + } + + static func from(error: Error, sourcekitd: SourceKitD) -> SKDResponse { + if let error = error as? SourceKitPluginError { + return error.response(sourcekitd: sourcekitd) + } else if error is CancellationError { + return SKDResponse(error: .cancelled, description: "Request cancelled", sourcekitd: sourcekitd) + } else { + return SKDResponse(error: .failed, description: String(describing: error), sourcekitd: sourcekitd) + } + } + + deinit { + sourcekitd.api.response_dispose(value) + } + + public func underlyingValueRetained() -> sourcekitd_api_response_t { + return sourcekitd.servicePluginApi.response_retain(value) + } + + public var description: String { + let cstr = sourcekitd.api.request_description_copy(value)! + defer { free(cstr) } + return String(cString: cstr) + } +} diff --git a/Sources/SwiftSourceKitPlugin/SKDResponseArrayBuilder.swift b/Sources/SwiftSourceKitPlugin/SKDResponseArrayBuilder.swift new file mode 100644 index 000000000..a5c208f37 --- /dev/null +++ b/Sources/SwiftSourceKitPlugin/SKDResponseArrayBuilder.swift @@ -0,0 +1,70 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Csourcekitd +import SourceKitD + +extension SourceKitD { + func responseArray(_ array: [SKDResponseValue]) -> SKDResponseArrayBuilder { + let result = SKDResponseArrayBuilder(sourcekitd: self) + for element in array { + result.append(element) + } + return result + } +} + +struct SKDResponseArrayBuilder { + /// The `SKDResponse` object that manages the lifetime of the `sourcekitd_response_t`. + private let response: SKDResponse + + var value: sourcekitd_api_response_t { response.value } + private var sourcekitd: SourceKitD { response.sourcekitd } + + init(sourcekitd: SourceKitD) { + response = .init( + takingUnderlyingResponse: sourcekitd.servicePluginApi.response_array_create(nil, 0), + sourcekitd: sourcekitd + ) + } + + func append(_ newValue: SKDResponseValue) { + switch newValue { + case let newValue as String: + sourcekitd.servicePluginApi.response_array_set_string(value, -1, newValue) + case is Bool: + preconditionFailure("Arrays of bools are not supported") + case let newValue as Int: + sourcekitd.servicePluginApi.response_array_set_int64(value, -1, Int64(newValue)) + case let newValue as Int64: + sourcekitd.servicePluginApi.response_array_set_int64(value, -1, newValue) + case let newValue as Double: + sourcekitd.servicePluginApi.response_array_set_double(value, -1, newValue) + case let newValue as sourcekitd_api_uid_t: + sourcekitd.servicePluginApi.response_array_set_uid(value, -1, newValue) + case let newValue as SKDResponseDictionaryBuilder: + sourcekitd.servicePluginApi.response_array_set_value(value, -1, newValue.value) + case let newValue as SKDResponseArrayBuilder: + sourcekitd.servicePluginApi.response_array_set_value(value, -1, newValue.value) + case let newValue as Array: + self.append(sourcekitd.responseArray(newValue)) + case let newValue as Dictionary: + self.append(sourcekitd.responseDictionary(newValue)) + case let newValue as Optional: + if let newValue { + self.append(newValue) + } + default: + preconditionFailure("Unknown type conforming to SKDRequestValue") + } + } +} diff --git a/Sources/SwiftSourceKitPlugin/SKDResponseDictionaryBuilder.swift b/Sources/SwiftSourceKitPlugin/SKDResponseDictionaryBuilder.swift new file mode 100644 index 000000000..48b4efd38 --- /dev/null +++ b/Sources/SwiftSourceKitPlugin/SKDResponseDictionaryBuilder.swift @@ -0,0 +1,81 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Csourcekitd +import SourceKitD +import SwiftSourceKitPluginCommon + +extension SourceKitD { + func responseDictionary(_ dict: [sourcekitd_api_uid_t: SKDResponseValue]) -> SKDResponseDictionaryBuilder { + let result = SKDResponseDictionaryBuilder(sourcekitd: self) + for (key, value) in dict { + result.set(key, to: value) + } + return result + } +} + +struct SKDResponseDictionaryBuilder { + /// The `SKDResponse` object that manages the lifetime of the `sourcekitd_response_t`. + let response: SKDResponse + + var value: sourcekitd_api_response_t { response.value } + private var sourcekitd: SourceKitD { response.sourcekitd } + + init(sourcekitd: SourceKitD) { + response = .init( + takingUnderlyingResponse: sourcekitd.servicePluginApi.response_dictionary_create(nil, nil, 0), + sourcekitd: sourcekitd + ) + } + + func set(_ key: sourcekitd_api_uid_t, to newValue: SKDResponseValue) { + switch newValue { + case let newValue as String: + sourcekitd.servicePluginApi.response_dictionary_set_string(value, key, newValue) + case let newValue as Bool: + sourcekitd.servicePluginApi.response_dictionary_set_bool(value, key, newValue) + case let newValue as Int: + sourcekitd.servicePluginApi.response_dictionary_set_int64(value, key, Int64(newValue)) + case let newValue as Int64: + sourcekitd.servicePluginApi.response_dictionary_set_int64(value, key, newValue) + case let newValue as Double: + sourcekitd.servicePluginApi.response_dictionary_set_double(value, key, newValue) + case let newValue as sourcekitd_api_uid_t: + sourcekitd.servicePluginApi.response_dictionary_set_uid(value, key, newValue) + case let newValue as SKDResponseDictionaryBuilder: + sourcekitd.servicePluginApi.response_dictionary_set_value(value, key, newValue.value) + case let newValue as SKDResponseArrayBuilder: + sourcekitd.servicePluginApi.response_dictionary_set_value(value, key, newValue.value) + case let newValue as Array: + self.set(key, to: sourcekitd.responseArray(newValue)) + case let newValue as Dictionary: + self.set(key, to: sourcekitd.responseDictionary(newValue)) + case let newValue as Optional: + if let newValue { + self.set(key, to: newValue) + } + default: + preconditionFailure("Unknown type conforming to SKDRequestValue") + } + } + + func set(_ key: sourcekitd_api_uid_t, toCustomBuffer buffer: UnsafeRawBufferPointer) { + assert(buffer.count > MemoryLayout.size, "custom buffer must begin with uint64_t identifier field") + sourcekitd.servicePluginApi.response_dictionary_set_custom_buffer( + value, + key, + buffer.baseAddress!, + buffer.count + ) + } +} diff --git a/Sources/SwiftSourceKitPlugin/SKDResponseValue.swift b/Sources/SwiftSourceKitPlugin/SKDResponseValue.swift new file mode 100644 index 000000000..57b999b5d --- /dev/null +++ b/Sources/SwiftSourceKitPlugin/SKDResponseValue.swift @@ -0,0 +1,32 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Csourcekitd + +/// Values that can be stored in a `SKDResponseDictionary` or `SKDResponseArray`. +/// +/// - Warning: `SKDResponseDictionary.set` and `SKDResponseDictionaryArray.append` +/// switch exhaustively over this protocol. +/// Do not add new conformances without adding a new case in the `set` and `append` functions. +protocol SKDResponseValue {} + +extension String: SKDResponseValue {} +extension Bool: SKDResponseValue {} +extension Int: SKDResponseValue {} +extension Int64: SKDResponseValue {} +extension Double: SKDResponseValue {} +extension sourcekitd_api_uid_t: SKDResponseValue {} +extension SKDResponseDictionaryBuilder: SKDResponseValue {} +extension SKDResponseArrayBuilder: SKDResponseValue {} +extension Array: SKDResponseValue {} +extension Dictionary: SKDResponseValue {} +extension Optional: SKDResponseValue where Wrapped: SKDResponseValue {} diff --git a/Sources/SwiftSourceKitPlugin/SourceKitDWrappers.swift b/Sources/SwiftSourceKitPlugin/SourceKitDWrappers.swift new file mode 100644 index 000000000..0c2b5c13d --- /dev/null +++ b/Sources/SwiftSourceKitPlugin/SourceKitDWrappers.swift @@ -0,0 +1,34 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Csourcekitd + +struct OpaqueIDEInspectionInstance: Sendable { + nonisolated(unsafe) let value: UnsafeMutableRawPointer + + internal init?(_ value: UnsafeMutableRawPointer?) { + guard let value else { + return nil + } + self.value = value + } +} + +struct RequestHandle: Sendable { + nonisolated(unsafe) let handle: sourcekitd_api_request_handle_t + internal init?(_ handle: sourcekitd_api_request_handle_t?) { + guard let handle else { + return nil + } + self.handle = handle + } +} diff --git a/Sources/SwiftSourceKitPlugin/SourceKitPluginError.swift b/Sources/SwiftSourceKitPlugin/SourceKitPluginError.swift new file mode 100644 index 000000000..ab3e3acf6 --- /dev/null +++ b/Sources/SwiftSourceKitPlugin/SourceKitPluginError.swift @@ -0,0 +1,32 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import SourceKitD + +/// An error that can be converted into an `SKDResponse``. +protocol SourceKitPluginError: Swift.Error { + func response(sourcekitd: SourceKitD) -> SKDResponse +} + +struct GenericPluginError: SourceKitPluginError { + let kind: SKDResponse.ErrorKind + let description: String + + internal init(kind: SKDResponse.ErrorKind = .failed, description: String) { + self.kind = kind + self.description = description + } + + func response(sourcekitd: SourceKitD) -> SKDResponse { + return SKDResponse(error: kind, description: description, sourcekitd: sourcekitd) + } +} diff --git a/Sources/SwiftSourceKitPluginCommon/CompletionResultsArray.swift b/Sources/SwiftSourceKitPluginCommon/CompletionResultsArray.swift new file mode 100644 index 000000000..dce46563e --- /dev/null +++ b/Sources/SwiftSourceKitPluginCommon/CompletionResultsArray.swift @@ -0,0 +1,232 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import SourceKitD + +#if compiler(>=6) +package import Csourcekitd +#else +import Csourcekitd +#endif + +package struct CompletionResult { + package struct StringEntry { + let start: UInt32 + + package init(start: UInt32) { + self.start = start + } + } + + let kind: sourcekitd_api_uid_t + let identifier: Int64 + let name: StringEntry + let description: StringEntry + let sourceText: StringEntry + let module: StringEntry? + let typename: StringEntry + let textMatchScore: Double + let semanticScore: Double + let semanticScoreComponents: StringEntry? + let priorityBucket: Int32 + let isSystemAndNumBytesToErase: UInt8 + let hasDiagnostic: Bool + let groupID: Int64 + + var isSystem: Bool { + isSystemAndNumBytesToErase & 0x80 != 0 + } + + var numBytesToErase: Int { + Int(isSystemAndNumBytesToErase & 0x7F) + } + + package init( + kind: sourcekitd_api_uid_t, + identifier: Int64, + name: StringEntry, + description: StringEntry, + sourceText: StringEntry, + module: StringEntry?, + typename: StringEntry, + textMatchScore: Double, + semanticScore: Double, + semanticScoreComponents: StringEntry?, + priorityBucket: Int32, + isSystem: Bool, + numBytesToErase: Int, + hasDiagnostic: Bool, + groupID: Int64 + ) { + self.kind = kind + self.identifier = identifier + self.name = name + self.description = description + self.sourceText = sourceText + self.module = module + self.typename = typename + self.textMatchScore = textMatchScore + self.semanticScore = semanticScore + self.semanticScoreComponents = semanticScoreComponents + self.priorityBucket = priorityBucket + precondition(numBytesToErase <= 0x7f, "numBytesToErase exceeds its storage") + self.isSystemAndNumBytesToErase = UInt8(numBytesToErase) & 0x7f | (isSystem ? 0x80 : 0) + self.hasDiagnostic = hasDiagnostic + self.groupID = groupID + } +} + +package struct VariantFunctions: Sendable { + nonisolated(unsafe) package let rawValue: sourcekitd_api_variant_functions_t +} + +package struct CompletionResultsArray { + let results: UnsafeBufferPointer + let strings: UnsafePointer + + init(pointer: UnsafeRawPointer) { + let numResults = pointer.load(fromByteOffset: 0, as: Int.self) + let resultStart = MemoryLayout.size + self.results = UnsafeBufferPointer.init( + start: (pointer + resultStart).assumingMemoryBound(to: CompletionResult.self), + count: numResults + ) + let stringStart = resultStart + results.count * MemoryLayout.stride + self.strings = (pointer + stringStart).assumingMemoryBound(to: CChar.self) + } + + init(_ variant: sourcekitd_api_variant_t) { + let ptr = UnsafeRawPointer(bitPattern: UInt(variant.data.1))! + self.init(pointer: ptr) + } + + private func cString(_ entry: CompletionResult.StringEntry) -> UnsafePointer { + return strings + Int(entry.start) + } + + private static func arrayGetCount(_ variant: sourcekitd_api_variant_t) -> Int { + return CompletionResultsArray(variant).results.count + } + + private static func arrayGetValue(_ variant: sourcekitd_api_variant_t, _ index: Int) -> sourcekitd_api_variant_t { + let results = CompletionResultsArray(variant) + precondition(index < results.results.count) + return sourcekitd_api_variant_t( + data: (UInt64(UInt(bitPattern: dictionaryFuncs.rawValue)), variant.data.1, UInt64(index)) + ) + } + + package static let arrayFuncs: VariantFunctions = { + let sourcekitd = DynamicallyLoadedSourceKitD.relativeToPlugin + let funcs = sourcekitd.pluginApi.variant_functions_create()! + sourcekitd.pluginApi.variant_functions_set_get_type(funcs, { _ in SOURCEKITD_API_VARIANT_TYPE_ARRAY }) + sourcekitd.pluginApi.variant_functions_set_array_get_count(funcs, { arrayGetCount($0) }) + sourcekitd.pluginApi.variant_functions_set_array_get_value(funcs, { arrayGetValue($0, $1) }) + return VariantFunctions(rawValue: funcs) + }() + + static func dictionaryApply( + _ dict: sourcekitd_api_variant_t, + _ applier: sourcekitd_api_variant_dictionary_applier_f_t?, + _ context: UnsafeMutableRawPointer? + ) -> Bool { + guard let applier else { + return true + } + + struct ApplierReturnedFalse: Error {} + + /// Calls `applier` and if `applier` returns `false`, throw `ApplierReturnedFalse`. + func apply( + _ key: sourcekitd_api_uid_t, + _ value: sourcekitd_api_variant_t, + _ context: UnsafeMutableRawPointer? + ) throws { + if !applier(key, value, context) { + throw ApplierReturnedFalse() + } + } + + let sourcekitd = DynamicallyLoadedSourceKitD.relativeToPlugin + let keys = sourcekitd.keys + + let results = CompletionResultsArray(dict) + let index = Int(dict.data.2) + + let result = results.results[index] + + do { + try apply(keys.kind, sourcekitd_api_variant_t(uid: result.kind), context) + try apply(keys.identifier, sourcekitd_api_variant_t(result.identifier), context) + try apply(keys.name, sourcekitd_api_variant_t(results.cString(result.name)), context) + try apply(keys.description, sourcekitd_api_variant_t(results.cString(result.description)), context) + try apply(keys.sourceText, sourcekitd_api_variant_t(results.cString(result.sourceText)), context) + if let module = result.module { + try apply(keys.moduleName, sourcekitd_api_variant_t(results.cString(module)), context) + } + try apply(keys.typeName, sourcekitd_api_variant_t(results.cString(result.typename)), context) + try apply(keys.priorityBucket, sourcekitd_api_variant_t(Int(result.priorityBucket)), context) + try apply(keys.textMatchScore, sourcekitd_api_variant_t(result.textMatchScore), context) + try apply(keys.semanticScore, sourcekitd_api_variant_t(result.semanticScore), context) + if let semanticScoreComponents = result.semanticScoreComponents { + try apply( + keys.semanticScoreComponents, + sourcekitd_api_variant_t(results.cString(semanticScoreComponents)), + context + ) + } + try apply(keys.isSystem, sourcekitd_api_variant_t(result.isSystem), context) + if result.numBytesToErase != 0 { + try apply(keys.numBytesToErase, sourcekitd_api_variant_t(result.numBytesToErase), context) + } + try apply(keys.hasDiagnostic, sourcekitd_api_variant_t(result.hasDiagnostic), context) + if (result.groupID != 0) { + try apply(keys.groupId, sourcekitd_api_variant_t(result.groupID), context) + } + } catch { + return false + } + return true + } + + static let dictionaryFuncs: VariantFunctions = { + let sourcekitd = DynamicallyLoadedSourceKitD.relativeToPlugin + let funcs = sourcekitd.pluginApi.variant_functions_create()! + sourcekitd.pluginApi.variant_functions_set_get_type(funcs, { _ in SOURCEKITD_API_VARIANT_TYPE_DICTIONARY }) + sourcekitd.pluginApi.variant_functions_set_dictionary_apply(funcs, { dictionaryApply($0, $1, $2) }) + return VariantFunctions(rawValue: funcs) + }() +} + +fileprivate extension sourcekitd_api_variant_t { + init(scalar: UInt64, type: sourcekitd_api_variant_type_t) { + self.init(data: (0, scalar, UInt64(type.rawValue))) + } + init(_ value: Int) { + self.init(scalar: UInt64(bitPattern: Int64(value)), type: SOURCEKITD_API_VARIANT_TYPE_INT64) + } + init(_ value: Int64) { + self.init(scalar: UInt64(bitPattern: value), type: SOURCEKITD_API_VARIANT_TYPE_INT64) + } + init(_ value: Bool) { + self.init(scalar: value ? 1 : 0, type: SOURCEKITD_API_VARIANT_TYPE_BOOL) + } + init(_ value: Double) { + self.init(scalar: value.bitPattern, type: SOURCEKITD_API_VARIANT_TYPE_DOUBLE) + } + init(_ value: UnsafePointer?) { + self.init(scalar: UInt64(UInt(bitPattern: value)), type: SOURCEKITD_API_VARIANT_TYPE_STRING) + } + init(uid: sourcekitd_api_uid_t) { + self.init(scalar: UInt64(UInt(bitPattern: uid)), type: SOURCEKITD_API_VARIANT_TYPE_UID) + } +} diff --git a/Sources/SwiftSourceKitPluginCommon/DynamicallyLoadedSourceKitD+relativeToPlugin.swift b/Sources/SwiftSourceKitPluginCommon/DynamicallyLoadedSourceKitD+relativeToPlugin.swift new file mode 100644 index 000000000..eb84b840a --- /dev/null +++ b/Sources/SwiftSourceKitPluginCommon/DynamicallyLoadedSourceKitD+relativeToPlugin.swift @@ -0,0 +1,58 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation +import SKLogging +import SwiftExtensions + +#if compiler(>=6) +package import SourceKitD +#else +import SourceKitD +#endif + +extension DynamicallyLoadedSourceKitD { + package static let relativeToPlugin: SourceKitD = { + var dlInfo = Dl_info() + dladdr(#dsohandle, &dlInfo) + let path = String(cString: dlInfo.dli_fname) + var url = URL(fileURLWithPath: path, isDirectory: false) + while url.pathExtension != "framework" && url.lastPathComponent != "/" { + url.deleteLastPathComponent() + } + url = + url + .deletingLastPathComponent() + .appendingPathComponent("sourcekitd.framework") + .appendingPathComponent("sourcekitd") + + let dlhandle: DLHandle + do { + dlhandle = try dlopen(url.filePath, mode: [.local, .first]) + } catch { + Logger(subsystem: "org.swift.sourcekit.client-plugin", category: "Loading") + .error("failed to find sourcekitd.framework relative to \(path); falling back to RTLD_DEFAULT") + dlhandle = .rtldDefault + } + + do { + return try DynamicallyLoadedSourceKitD( + dlhandle: dlhandle, + path: URL(string: "fake://")!, + pluginPaths: nil, + initialize: false + ) + } catch { + fatalError("Failed to load sourcekitd: \(error)") + } + }() +} diff --git a/Tests/CompletionScoringPerfTests/CandidateBatchPerfTests.swift b/Tests/CompletionScoringPerfTests/CandidateBatchPerfTests.swift new file mode 100644 index 000000000..bbebc2549 --- /dev/null +++ b/Tests/CompletionScoringPerfTests/CandidateBatchPerfTests.swift @@ -0,0 +1,52 @@ +// +// CandidateBatchPerfTests.swift +// CompletionScoringPerfTests +// +// Created by Alex Hoppen on 21.2.22. +// + +import CompletionScoring +import CompletionScoringTestSupport +import XCTest + +class CandidateBatchPerfTests: XCTestCase { + func testCandidateBatchCreation() { + gaugeTiming { + var candidateBatch = CandidateBatch() + for _ in 1..<100_000 { + candidateBatch.append("aAAAAAAAAAaAAAAAAAAAaAAAAAAAAA", contentType: .codeCompletionSymbol) + } + } + } + + func testCandidateBatchBulkLoading() { + typealias UTF8Bytes = Pattern.UTF8Bytes + var randomness = RepeatableRandomNumberGenerator() + let typeStrings = (0..<100_000).map { _ in + SymbolGenerator.shared.randomType(using: &randomness) + } + let typeUTF8Buffers = typeStrings.map { typeString in + typeString.allocateCopyOfUTF8Buffer() + }; + defer { + for typeUTF8Buffer in typeUTF8Buffers { + typeUTF8Buffer.deallocate() + } + } + + gaugeTiming(iterations: 10) { + drain(CandidateBatch(candidates: typeUTF8Buffers, contentType: .unknown)) + } + + // A baseline for what this method replaced, initial commit had the replacement running in 2/3rds the time of this. + #if false + gaugeTiming(iterations: 10) { + var batch = CandidateBatch() + for typeUTF8Buffer in typeUTF8Buffers { + batch.append(typeUTF8Buffer, contentType: .unknown) + } + drain(batch) + } + #endif + } +} diff --git a/Tests/CompletionScoringPerfTests/ScoringPerfTests.Completion.swift b/Tests/CompletionScoringPerfTests/ScoringPerfTests.Completion.swift new file mode 100644 index 000000000..0200c7c65 --- /dev/null +++ b/Tests/CompletionScoringPerfTests/ScoringPerfTests.Completion.swift @@ -0,0 +1,23 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import CompletionScoring +import Foundation + +extension ScoringPerfTests { + struct Completion { + var filterText: String + var displayText: String + var semanticClassification: SemanticClassification + var groupID: Int? + } +} diff --git a/Tests/CompletionScoringPerfTests/ScoringPerfTests.Corpus.swift b/Tests/CompletionScoringPerfTests/ScoringPerfTests.Corpus.swift new file mode 100644 index 000000000..dd83062d3 --- /dev/null +++ b/Tests/CompletionScoringPerfTests/ScoringPerfTests.Corpus.swift @@ -0,0 +1,65 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import CompletionScoring +import CompletionScoringTestSupport +import Foundation + +extension ScoringPerfTests { + struct Corpus { + let providers: [Provider] + let patterns: [Pattern] + let batches: [CandidateBatch] + let tokenizedInfluencers: [[String]] + let totalCandidates: Int + + init(patternPrefixLengths: Range) { + var randomness = RepeatableRandomNumberGenerator() + + var nextGroupID = 0 + #if DEBUG + self.providers = [ + Provider.sdkProvider(randomness: &randomness, nextGroupID: &nextGroupID) // Like SourceKit + ] + #else + self.providers = [ + Provider.sdkProvider(randomness: &randomness, nextGroupID: &nextGroupID), // Like SourceKit + Provider.sdkProvider(randomness: &randomness, nextGroupID: &nextGroupID), // Like SymbolCache + Provider.sdkProvider(randomness: &randomness, nextGroupID: &nextGroupID), // Like AllSymbols + Provider.snippetProvider(randomness: &randomness), + ] + #endif + let symbolGenerator = SymbolGenerator.shared + + self.batches = providers.map(\.batch) + self.patterns = (0..<10).flatMap { _ -> [Pattern] in + let patternText = symbolGenerator.randomPatternText( + lengthRange: patternPrefixLengths, + using: &randomness + ) + return (patternPrefixLengths.lowerBound...patternText.count).map { length in + Pattern(text: String(patternText.prefix(length))) + } + } + + self.tokenizedInfluencers = MatchCollator.tokenize( + influencingTokenizedIdentifiers: [ + "editorController", + "view", + ], + filterLowSignalTokens: true + ) + + self.totalCandidates = providers.map(\.candidates.count).sum() + } + } +} diff --git a/Tests/CompletionScoringPerfTests/ScoringPerfTests.Module.swift b/Tests/CompletionScoringPerfTests/ScoringPerfTests.Module.swift new file mode 100644 index 000000000..353ea1bf6 --- /dev/null +++ b/Tests/CompletionScoringPerfTests/ScoringPerfTests.Module.swift @@ -0,0 +1,117 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import CompletionScoring +import CompletionScoringTestSupport +import Foundation + +extension ScoringPerfTests { + struct Module { + var completions: [Completion] + init( + randomness: inout RepeatableRandomNumberGenerator, + moduleProximity: ModuleProximity, + nextGroupID: inout Int + ) { + let globalFunctionCount = (5..<50).randomElement(using: &randomness)! + let globalTypesCount = (5..<250).randomElement(using: &randomness)! + let weightedAvailability = WeightedChoices([ + (0.01, .deprecated), + (0.05, .unavailable), + (0.85, .available), + ]) + + let weightedTypeCompatibility = WeightedChoices([ + (0.10, .invalid), + (0.05, .compatible), + (0.85, .unrelated), + ]) + + let functionCompletions: [Completion] = Array(count: globalFunctionCount) { + let function = SymbolGenerator.shared.randomFunction(using: &randomness) + let typeCompatibility = weightedTypeCompatibility.select(using: &randomness) + let availability = weightedAvailability.select(using: &randomness) + let classification = SemanticClassification( + availability: availability, + completionKind: .function, + flair: [], + moduleProximity: moduleProximity, + popularity: .none, + scopeProximity: .global, + structuralProximity: .inapplicable, + synchronicityCompatibility: .compatible, + typeCompatibility: typeCompatibility + ) + return Completion( + filterText: function.filterText, + displayText: function.displayText, + semanticClassification: classification + ) + } + + let typeCompletions: [Completion] = Array(count: globalTypesCount) { + let text = SymbolGenerator.shared.randomType(using: &randomness) + let typeCompatibility = weightedTypeCompatibility.select(using: &randomness) + let availability = weightedAvailability.select(using: &randomness) + let classification = SemanticClassification( + availability: availability, + completionKind: .function, + flair: [], + moduleProximity: moduleProximity, + popularity: .none, + scopeProximity: .global, + structuralProximity: .inapplicable, + synchronicityCompatibility: .compatible, + typeCompatibility: typeCompatibility + ) + let groupID = nextGroupID + nextGroupID += 1 + return Completion( + filterText: text, + displayText: text, + semanticClassification: classification, + groupID: groupID + ) + } + + let initializers: [Completion] = typeCompletions.flatMap { typeCompletion -> [Completion] in + let initializers = SymbolGenerator.shared.randomInitializers( + typeName: typeCompletion.filterText, + using: &randomness + ) + return initializers.map { initializer -> Completion in + let typeCompatibility = weightedTypeCompatibility.select(using: &randomness) + let availability = weightedAvailability.select(using: &randomness) + let classification = SemanticClassification( + availability: availability, + completionKind: .initializer, + flair: [], + moduleProximity: moduleProximity, + popularity: .none, + scopeProximity: .global, + structuralProximity: .inapplicable, + synchronicityCompatibility: .compatible, + typeCompatibility: typeCompatibility + ) + return Completion( + filterText: initializer.filterText, + displayText: initializer.displayText, + semanticClassification: classification, + groupID: typeCompletion.groupID + ) + } + } + + self.completions = typeCompletions + functionCompletions + initializers + } + } +} diff --git a/Tests/CompletionScoringPerfTests/ScoringPerfTests.Provider.swift b/Tests/CompletionScoringPerfTests/ScoringPerfTests.Provider.swift new file mode 100644 index 000000000..e086f0302 --- /dev/null +++ b/Tests/CompletionScoringPerfTests/ScoringPerfTests.Provider.swift @@ -0,0 +1,72 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import CompletionScoring +import CompletionScoringTestSupport +import Foundation + +extension ScoringPerfTests { + struct Provider { + let candidates: [Completion] + let batch: CandidateBatch + + init(candidates: [Completion]) { + self.candidates = candidates + self.batch = CandidateBatch(symbols: candidates.map(\.filterText)) + } + + static func sdkProvider(randomness: inout RepeatableRandomNumberGenerator, nextGroupID: inout Int) -> Self { + let moduleCount = 1 * ScoringPerfTests.scale + let moduleProximities = WeightedChoices([ + (1 / 16.0, .same), + (1 / 16.0, .imported(distance: 0)), + (2 / 16.0, .imported(distance: 1)), + (4 / 16.0, .imported(distance: 2)), + (8 / 16.0, .imported(distance: 3)), + ]) + + let modules = Array(count: moduleCount) { + Module( + randomness: &randomness, + moduleProximity: moduleProximities.select(using: &randomness), + nextGroupID: &nextGroupID + ) + } + return Self(candidates: modules.flatMap(\.completions)) + } + + static func snippetProvider(randomness: inout RepeatableRandomNumberGenerator) -> Self { + + return Self( + candidates: Array(count: 100) { + let title = SymbolGenerator.shared.randomSegment(using: &randomness, capitalizeFirstTerm: false) + let classification = SemanticClassification( + availability: .inapplicable, + completionKind: .unspecified, + flair: [], + moduleProximity: .inapplicable, + popularity: .none, + scopeProximity: .inapplicable, + structuralProximity: .inapplicable, + synchronicityCompatibility: .inapplicable, + typeCompatibility: .inapplicable + ) + return Completion( + filterText: title, + displayText: "\(title) - Code Snippet", + semanticClassification: classification + ) + } + ) + } + } +} diff --git a/Tests/CompletionScoringPerfTests/ScoringPerfTests.swift b/Tests/CompletionScoringPerfTests/ScoringPerfTests.swift new file mode 100644 index 000000000..69ef73411 --- /dev/null +++ b/Tests/CompletionScoringPerfTests/ScoringPerfTests.swift @@ -0,0 +1,213 @@ +// +// ScoringPerfTests.swift +// CompletionScoringPerfTests +// +// Created by Ben Langmuir on 1/25/21. +// + +import CompletionScoring +import CompletionScoringTestSupport +import XCTest + +typealias Pattern = CompletionScoring.Pattern + +class ScoringPerfTests: XCTestCase { + #if DEBUG + static let scale = 1 + #else + static let scale = 100 + #endif + var scale: Int { Self.scale } + + private enum Scenario { + case fastScoringOnly + case fullScoringAndCutoff + case fullScoringAndCutoffWithInfluencers + case fullScoringAndCutoffWithInfluencersAndSingleLetterFilter + case fullScoringAndCutoffWithoutInfluencersAndSingleLetterFilter + + var selectBest: Bool { + switch self { + case .fastScoringOnly: return false + case .fullScoringAndCutoff: return true + case .fullScoringAndCutoffWithInfluencers: return true + case .fullScoringAndCutoffWithInfluencersAndSingleLetterFilter: return true + case .fullScoringAndCutoffWithoutInfluencersAndSingleLetterFilter: return true + } + } + + var usesInfluencers: Bool { + switch self { + case .fastScoringOnly: return false + case .fullScoringAndCutoff: return false + case .fullScoringAndCutoffWithInfluencers: return true + case .fullScoringAndCutoffWithInfluencersAndSingleLetterFilter: return true + case .fullScoringAndCutoffWithoutInfluencersAndSingleLetterFilter: return true + } + } + + var patternPrefixLengths: Range { + switch self { + case .fastScoringOnly: return 0..<16 + case .fullScoringAndCutoff: return 0..<16 + case .fullScoringAndCutoffWithInfluencers: return 0..<16 + case .fullScoringAndCutoffWithInfluencersAndSingleLetterFilter: return 1..<2 + case .fullScoringAndCutoffWithoutInfluencersAndSingleLetterFilter: return 1..<2 + } + } + } + + func testFastConcurrentScoringPerformance() throws { + testConcurrentScoringAndBatchingPerformance(scenario: .fastScoringOnly) + } + + func testFullScoringAndSelectionPerformanceWithoutInfluencers() throws { + testConcurrentScoringAndBatchingPerformance(scenario: .fullScoringAndCutoff) + } + + func testFullScoringAndSelectionPerformanceWithInfluencers() throws { + testConcurrentScoringAndBatchingPerformance(scenario: .fullScoringAndCutoffWithInfluencers) + } + + func testFullScoringAndSelectionPerformanceWithInfluencersAndSingleLetterFilter() throws { + testConcurrentScoringAndBatchingPerformance(scenario: .fullScoringAndCutoffWithInfluencersAndSingleLetterFilter) + } + + func testFullScoringAndSelectionPerformanceWithoutInfluencersAndSingleLetterFilter() throws { + testConcurrentScoringAndBatchingPerformance( + scenario: .fullScoringAndCutoffWithoutInfluencersAndSingleLetterFilter + ) + } + + private func testConcurrentScoringAndBatchingPerformance(scenario: Scenario) { + let corpus = Corpus(patternPrefixLengths: scenario.patternPrefixLengths) + let batches = corpus.batches + let totalCandidates = corpus.totalCandidates + let sessions: [[Pattern]] = { + var patternGroups: [[Pattern]] = [] + for pattern in corpus.patterns { + if let previous = patternGroups.last?.last, pattern.text.hasPrefix(previous.text) { + patternGroups[patternGroups.count - 1].append(pattern) + } else { + patternGroups.append([pattern]) + } + } + return patternGroups + }() + + var scoredCandidates = 0 + var matchedCandidates = 0 + var selectedCandidates = 0 + var attempts = 0 + let tokenizedInfluencers = scenario.usesInfluencers ? corpus.tokenizedInfluencers : [] + gaugeTiming { + for session in sessions { + let selector = ScoredMatchSelector(batches: batches) + for pattern in session { + let matches = selector.scoredMatches(pattern: pattern, precision: .fast) + if scenario.selectBest { + let fastMatchReps: [MatchCollator.Match] = matches.map { match in + let candidate = corpus.providers[match.batchIndex].candidates[match.candidateIndex] + let score = CompletionScore( + textComponent: match.textScore, + semanticComponent: candidate.semanticClassification.score + ) + return MatchCollator.Match( + batchIndex: match.batchIndex, + candidateIndex: match.candidateIndex, + groupID: candidate.groupID, + score: score + ) + } + let bestMatches = MatchCollator.selectBestMatches( + for: pattern, + from: fastMatchReps, + in: batches, + influencingTokenizedIdentifiers: tokenizedInfluencers + ) { lhs, rhs in + let lhsCandidate = corpus.providers[lhs.batchIndex].candidates[lhs.candidateIndex] + let rhsCandidate = corpus.providers[rhs.batchIndex].candidates[rhs.candidateIndex] + return lhsCandidate.displayText < rhsCandidate.displayText + } + selectedCandidates += bestMatches.count + } + matchedCandidates += matches.count + scoredCandidates += totalCandidates + attempts += 1 + } + } + } + print("> Sessions: \(sessions.map {$0.map(\.text)})") + print("> Candidates: \(scoredCandidates)") + print("> Matches: \(matchedCandidates)") + print("> Selected: \(selectedCandidates)") + print("> Attempts: \(attempts)") + } + + func testThoroughConcurrentScoringPerformance() throws { + let corpus = Corpus( + patternPrefixLengths: MatchCollator.minimumPatternLengthToAlwaysRescoreWithThoroughPrecision..<16 + ) + let batches = corpus.batches + + struct MatchSet { + var pattern: Pattern + var fastMatchReps: [MatchCollator.Match] + } + + let matchSets: [MatchSet] = corpus.patterns.map { pattern in + let matches = pattern.scoredMatches(across: batches, precision: .fast) + let fastMatchReps = matches.map { match in + let candidate = corpus.providers[match.batchIndex].candidates[match.candidateIndex] + let score = CompletionScore( + textComponent: match.textScore, + semanticComponent: candidate.semanticClassification.score + ) + return MatchCollator.Match( + batchIndex: match.batchIndex, + candidateIndex: match.candidateIndex, + groupID: candidate.groupID, + score: score + ) + } + return MatchSet(pattern: pattern, fastMatchReps: fastMatchReps) + } + gaugeTiming { + for matchSet in matchSets { + let bestMatches = MatchCollator.selectBestMatches( + for: matchSet.pattern, + from: matchSet.fastMatchReps, + in: batches, + influencingTokenizedIdentifiers: [] + ) { lhs, rhs in + return false + } + drain(bestMatches) + } + } + } + + func testInfluencingIdentifiersInIsolation() { + let corpus = Corpus(patternPrefixLengths: Scenario.fullScoringAndCutoffWithInfluencers.patternPrefixLengths) + gaugeTiming { + drain( + MatchCollator.performanceTest_influenceScores( + for: corpus.batches, + influencingTokenizedIdentifiers: corpus.tokenizedInfluencers, + iterations: 350 + ) + ) + } + } + + func testTokenizingInIsolation() { + let corpus = Corpus(patternPrefixLengths: Scenario.fullScoringAndCutoffWithInfluencers.patternPrefixLengths) + gaugeTiming { + for pattern in corpus.patterns { + for batch in corpus.batches { + drain(pattern.testPerformance_tokenizing(batch: batch, contentType: .codeCompletionSymbol)) + } + } + } + } +} diff --git a/Tests/CompletionScoringPerfTests/Supporting Files/CodeCompletionFoundationPerfTests-Info.plist b/Tests/CompletionScoringPerfTests/Supporting Files/CodeCompletionFoundationPerfTests-Info.plist new file mode 100644 index 000000000..64d65ca49 --- /dev/null +++ b/Tests/CompletionScoringPerfTests/Supporting Files/CodeCompletionFoundationPerfTests-Info.plist @@ -0,0 +1,22 @@ + + + + + CFBundleDevelopmentRegion + $(DEVELOPMENT_LANGUAGE) + CFBundleExecutable + $(EXECUTABLE_NAME) + CFBundleIdentifier + $(PRODUCT_BUNDLE_IDENTIFIER) + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + $(PRODUCT_NAME) + CFBundlePackageType + $(PRODUCT_BUNDLE_PACKAGE_TYPE) + CFBundleShortVersionString + 1.0 + CFBundleVersion + 1 + + diff --git a/Tests/CompletionScoringTests/BinaryCodingTests.swift b/Tests/CompletionScoringTests/BinaryCodingTests.swift new file mode 100644 index 000000000..ad7379154 --- /dev/null +++ b/Tests/CompletionScoringTests/BinaryCodingTests.swift @@ -0,0 +1,168 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import CompletionScoring +import Foundation +import XCTest + +class BinaryCodingTests: XCTestCase { + func roundTrip(_ encoded: V) throws -> V { + try V(binaryCodedRepresentation: encoded.binaryCodedRepresentation(contentVersion: 0)) + } + + func testRoundTripping(_ encoded: some BinaryCodable & Equatable) { + XCTAssertNoThrow( + try { + let decoded = try roundTrip(encoded) + XCTAssertEqual(encoded, decoded) + }() + ) + } + + func testRoundTrippingAll(_ encodedValues: [some BinaryCodable & Equatable]) { + for encodedValue in encodedValues { + testRoundTripping(encodedValue) + } + } + + func testIntegers() { + func test(_ type: I.Type) { + testRoundTrippingAll([I.min, I(exactly: -1), I.zero, I(exactly: 1), I.max]) + } + + test(Int.self) + test(Int8.self) + test(Int16.self) + test(Int32.self) + test(Int64.self) + test(UInt.self) + test(UInt8.self) + test(UInt16.self) + test(UInt32.self) + test(UInt64.self) + } + + func testFloats() { + func test(_ type: F.Type) { + let values = [ + F.zero, F.leastNonzeroMagnitude, F.leastNormalMagnitude, F(1), F.greatestFiniteMagnitude, F.infinity, + ] + for value in values { + testRoundTripping(value) + testRoundTripping(-value) + } + let decodedNan = try? roundTrip(F.nan) + XCTAssert(decodedNan?.isNaN == true) + } + test(Float.self) + test(Double.self) + } + + func testBools() { + testRoundTrippingAll([false, true]) + } + + func testStrings() { + testRoundTripping("a") + testRoundTrippingAll(["", "a", "é", " ", "\n", "aa"]) + testRoundTrippingAll(["🤷", "🤷🏻", "🤷🏼", "🤷🏽", "🤷🏾", "🤷🏿", "🤷‍♀️", "🤷🏻‍♀️", "🤷🏼‍♀️", "🤷🏽‍♀️", "🤷🏾‍♀️", "🤷🏿‍♀️", "🤷‍♂️", "🤷🏻‍♂️", "🤷🏼‍♂️", "🤷🏽‍♂️", "🤷🏾‍♂️", "🤷🏿‍♂️"]) + } + + func testArrays() { + testRoundTrippingAll([[], [0], [0, 1]]) + } + + func testDictionaries() { + testRoundTrippingAll([[:], [0: false], [0: false, 1: true]]) + } + + func testSemanticClassificationComponents() { + testRoundTrippingAll([.keyword] as [CompletionKind]) + testRoundTrippingAll( + [ + .keyword, .enumCase, .variable, .function, .initializer, .argumentLabels, .type, .other, .unknown, + .unspecified, + ] as [CompletionKind] + ) + testRoundTrippingAll([.chainedMember, .commonKeywordAtCurrentPosition] as [Flair]) + testRoundTrippingAll( + [ + .imported(distance: 0), .imported(distance: 1), .importable, .inapplicable, .unknown, .invalid, + .unspecified, + ] as [ModuleProximity] + ) + testRoundTrippingAll([.none, .unspecified] as [Popularity]) + testRoundTrippingAll( + [ + .local, .argument, .container, .inheritedContainer, .outerContainer, .global, .inapplicable, .unknown, + .unspecified, + ] as [ScopeProximity] + ) + testRoundTrippingAll( + [.project(fileSystemHops: nil), .project(fileSystemHops: 1), .sdk, .inapplicable, .unknown, .unspecified] + as [StructuralProximity] + ) + testRoundTrippingAll( + [.compatible, .convertible, .incompatible, .inapplicable, .unknown, .unspecified] + as [SynchronicityCompatibility] + ) + testRoundTrippingAll( + [.compatible, .unrelated, .invalid, .inapplicable, .unknown, .unspecified] as [TypeCompatibility] + ) + testRoundTrippingAll( + [.available, .softDeprecated, .deprecated, .unknown, .inapplicable, .unspecified] as [Availability] + ) + testRoundTripping( + SemanticClassification( + availability: .softDeprecated, + completionKind: .function, + flair: .chainedCallToSuper, + moduleProximity: .importable, + popularity: .unspecified, + scopeProximity: .container, + structuralProximity: .sdk, + synchronicityCompatibility: .convertible, + typeCompatibility: .compatible + ) + ) + testRoundTripping( + SemanticClassification( + availability: .deprecated, + completionKind: .type, + flair: .chainedMember, + moduleProximity: .imported(distance: 2), + popularity: .none, + scopeProximity: .global, + structuralProximity: .project(fileSystemHops: 4), + synchronicityCompatibility: .compatible, + typeCompatibility: .unrelated + ) + ) + } +} + +protocol FiniteInteger: BinaryInteger, BinaryCodable { + static var min: Self { get } + static var max: Self { get } + static var zero: Self { get } +} + +extension Int: FiniteInteger {} +extension Int8: FiniteInteger {} +extension Int16: FiniteInteger {} +extension Int32: FiniteInteger {} +extension Int64: FiniteInteger {} +extension UInt: FiniteInteger {} +extension UInt8: FiniteInteger {} +extension UInt16: FiniteInteger {} +extension UInt32: FiniteInteger {} +extension UInt64: FiniteInteger {} diff --git a/Tests/CompletionScoringTests/CandidateBatchTests.swift b/Tests/CompletionScoringTests/CandidateBatchTests.swift new file mode 100644 index 000000000..90a22ffae --- /dev/null +++ b/Tests/CompletionScoringTests/CandidateBatchTests.swift @@ -0,0 +1,1478 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import CompletionScoring +import CompletionScoringTestSupport +import XCTest + +typealias Pattern = CompletionScoring.Pattern + +class CandidateBatchTests: XCTestCase { + func testEnumerate() throws { + let strings = [ + "abc", + "foo(bar:baz:)?", + "", + "ASomewhatLongStringAtLeastLongEnoughToNotBeASmolStringInSwift", + ] + + let batch = CandidateBatch(symbols: strings) + var index = 0 + batch.enumerate { candidate in + XCTAssertEqual(strings[index], String(bytes: candidate.bytes, encoding: .utf8)!) + index += 1 + } + index = 0 + batch.enumerate { candidateIndex, candidate in + XCTAssertEqual(candidateIndex, index) + XCTAssertEqual(strings[index], String(bytes: candidate.bytes, encoding: .utf8)!) + index += 1 + } + XCTAssertEqual(index, strings.count) + } + + func testCount() throws { + XCTAssertEqual(CandidateBatch(symbols: []).count, 0) + XCTAssertEqual(CandidateBatch(symbols: [""]).count, 1) + XCTAssertEqual(CandidateBatch(symbols: ["", ""]).count, 2) + } + + func testStringAt() throws { + XCTAssertEqual(CandidateBatch(symbols: []).count, 0) + XCTAssertEqual(CandidateBatch(symbols: [""])[stringAt: 0], "") + XCTAssertEqual(CandidateBatch(symbols: ["A"])[stringAt: 0], "A") + XCTAssertEqual(CandidateBatch(symbols: ["", "A"])[stringAt: 0], "") + XCTAssertEqual(CandidateBatch(symbols: ["", "A"])[stringAt: 1], "A") + } + + func testAppending() throws { + var randomness = RepeatableRandomNumberGenerator() + var strings: [String] = [] + var batch = CandidateBatch(byteCapacity: 1) // We want to thoroughly test growing, so start very small. + for _ in 0..<2500 { + let string = randomness.randomLowercaseASCIIString(lengthRange: 0...11) + strings.append(string) + batch.append(string, contentType: .codeCompletionSymbol) + XCTAssertEqual(strings.first, batch[stringAt: 0]) + XCTAssertEqual(strings.last, batch[stringAt: batch.count - 1]) + } + batch.enumerate { index, candidate in + XCTAssertEqual(strings[index], String(bytes: candidate.bytes, encoding: .utf8)!) + } + } + + func testCopyOnWrite() throws { + let initialStrings = ["ABC"] + var original = CandidateBatch(symbols: initialStrings) + var copy = original + XCTAssertEqual(copy.strings, initialStrings) + XCTAssertEqual(original.strings, initialStrings) + + original.append("123", contentType: .codeCompletionSymbol) + XCTAssertEqual(copy.strings, initialStrings) + XCTAssertEqual(original.strings, initialStrings + ["123"]) + + copy.append("EFG", contentType: .codeCompletionSymbol) + XCTAssertEqual(copy.strings, initialStrings + ["EFG"]) + XCTAssertEqual(original.strings, initialStrings + ["123"]) + } + + // This test tries to find some breakdown with copy-on-write by racing two threads against each other. + // One thread does a read while the other days a write. + func testConcurrentMutation() { + @Sendable func validate(_ candidate: Candidate) { + let count = candidate.bytes.count + for byte in candidate.bytes { + XCTAssertEqual(Int(byte), count) + } + } + @Sendable func read(local: CandidateBatch) { + local.withUnsafeStorage { storage in + if let candidateIndex = storage.indices.randomElement() { + validate(storage.candidate(at: candidateIndex)) + } + } + } + for _ in 0..<100 { // Inner loop goes N^2 with copy on write, this makes it only go linearly slower. + // `nonisolated(unsafe)` is fine because there is only one operation to `shared` before a + // `DispatchGroup.wait` and thus we can't have concurrent accesses. + nonisolated(unsafe) var shared = CandidateBatch() + let queue = DispatchQueue(label: "", attributes: .concurrent) + let strings = UTF8Byte.uppercaseAZ.map { letter in + Array(repeating: String(format: "%c", letter), count: Int(letter)).joined() + } + // Try reading the shared value while writing into a copy. + for _ in 0..<100 { + // Having both operated directly on the `shared` captured through the closure would definitely be + // illegal, and would crash, just like it does for Swift.Array + let captured = shared + let group = DispatchGroup() + queue.async(group: group) { + read(local: captured) + } + queue.async(group: group) { + shared.append(strings.randomElement()!, contentType: .codeCompletionSymbol) + } + group.wait() + } + // Try reading a copy while writing into the shared value. + for _ in 0..<100 { + let group = DispatchGroup() + queue.async(group: group) { + read(local: shared) + } + // `nonisolated(unsafe)` is fine because there is only one operation to `capture` before a + // `DispatchGroup.wait` and thus we can't have concurrent accesses. + nonisolated(unsafe) var captured = shared + queue.async(group: group) { + captured.append(strings.randomElement()!, contentType: .codeCompletionSymbol) + } + group.wait() + } + } + } + + private func randomBatch( + count: Int, + maxStringLength: Int, + using randomness: inout RepeatableRandomNumberGenerator + ) -> CandidateBatch { + CandidateBatch( + symbols: (0.. [String] { + let matches = Pattern(text: pattern).scoredMatches(in: batch, precision: .fast).sorted { lhs, rhs in + lhs.textScore > rhs.textScore + } + return matches.map { match in + batch[stringAt: match.candidateIndex] + } + } + + XCTAssertEqual(match(pattern: "a"), ["a", "aa", "aaa", "aabb"]) + XCTAssertEqual(match(pattern: "aa"), ["aa", "aaa", "aabb"]) + XCTAssertEqual(match(pattern: "aaa"), ["aaa"]) + + XCTAssertEqual(match(pattern: "b"), ["b", "bb", "aabb"]) + XCTAssertEqual(match(pattern: "bb"), ["bb", "aabb"]) + + XCTAssertEqual(match(pattern: "ab"), ["aabb"]) + } + + func testFilter() { + let batch = CandidateBatch(symbols: [ + "a", + "aabb", + "aa", + "aaa", + "bb", + "b", + ]) + let filtered = batch.filter { candidateIndex, candidate in + candidate.bytes.count == 2 + } + XCTAssertEqual(filtered.strings, ["aa", "bb"]) + } + + private func bestMatches( + _ filterText: String, + _ matches: [SemanticScoredText], + maximumNumberOfItemsForExpensiveSelection: Int = MatchCollator.defaultMaximumNumberOfItemsForExpensiveSelection + ) -> [String] { + let pattern = Pattern(text: filterText) + let batch = CandidateBatch(candidates: matches) + let batchMatches = pattern.scoredMatches(across: [batch], precision: .fast) + let fastMatches = batchMatches.map { batchMatch in + MatchCollator.Match( + batchIndex: batchMatch.batchIndex, + candidateIndex: batchMatch.candidateIndex, + groupID: matches[batchMatch.candidateIndex].groupID, + score: CompletionScore( + textComponent: batchMatch.textScore, + semanticComponent: matches[batchMatch.candidateIndex].semanticScore + ) + ) + } + let bestMatches = MatchCollator.selectBestMatches( + for: pattern, + from: fastMatches, + in: [batch], + influencingTokenizedIdentifiers: [], + orderingTiesBy: { _, _ in false }, + maximumNumberOfItemsForExpensiveSelection: maximumNumberOfItemsForExpensiveSelection + ) + return bestMatches.map { bestMatch in + matches[bestMatch.candidateIndex].text + } + } + + fileprivate func bestMatches(_ filterText: String, _ candidates: [String]) -> [String] { + bestMatches( + filterText, + candidates.map { candidate in + SemanticScoredText(candidate) + } + ) + } + + func testEarlySemanticCutoff() { + for isMainDotSwift in [true, false] { + let typeFlair: Flair = isMainDotSwift ? [] : .expressionAtNonScriptOrMainFileScope + let best = bestMatches( + "s", + [ + SemanticScoredText( + "struct", + SemanticClassification( + availability: .available, + completionKind: .keyword, + flair: [.commonKeywordAtCurrentPosition], + moduleProximity: .inapplicable, + popularity: .none, + scopeProximity: .inapplicable, + structuralProximity: .inapplicable, + synchronicityCompatibility: .inapplicable, + typeCompatibility: .inapplicable + ).score + ), + SemanticScoredText( + "String", + SemanticClassification( + availability: .available, + completionKind: .type, + flair: [typeFlair], + moduleProximity: .imported(distance: 0), + popularity: .none, + scopeProximity: .global, + structuralProximity: .inapplicable, + synchronicityCompatibility: .inapplicable, + typeCompatibility: .inapplicable + ).score + ), + SemanticScoredText( + "sint8", + SemanticClassification( + availability: .available, + completionKind: .type, + flair: [typeFlair], + moduleProximity: .imported(distance: 2), + popularity: .none, + scopeProximity: .global, + structuralProximity: .inapplicable, + synchronicityCompatibility: .inapplicable, + typeCompatibility: .inapplicable + ).score + ), + ] + ) + if isMainDotSwift { + XCTAssertEqual(best, ["struct", "String", "sint8"]) + } else { + XCTAssertEqual(best, ["struct"]) + } + } + } + + func testEmptyPatterns() { + withEachPermutation("az", "ay") { first, second in + XCTAssertEqual(bestMatches("", [first, second]), ["ay", "az"]) + } + + withEachPermutation(SemanticScoredText("foo", 2.0), SemanticScoredText("bar", 1.0)) { first, second in + XCTAssertEqual(bestMatches("", [first, second]), ["foo", "bar"]) + } + + withEachPermutation(SemanticScoredText("foo", 1.0), SemanticScoredText("bar", 2.0)) { first, second in + XCTAssertEqual(bestMatches("", [first, second]), ["bar", "foo"]) + } + } + + func testBestMatchesTieOrder() { + withEachPermutation("az", "ay") { first, second in + XCTAssertEqual(bestMatches("a", [first, second]), ["ay", "az"]) + } + } + + func testBestMatches() { + XCTAssertEqual(bestMatches("a", ["a"]), ["a"]) + XCTAssertEqual(bestMatches("b", ["a"]), []) + + XCTAssertEqual(bestMatches("a", ["a", "bab"]), ["a"]) + XCTAssertEqual(bestMatches("a", ["bab", "a"]), ["a"]) + + XCTAssertEqual(bestMatches("a", ["a", "c"]), ["a"]) + XCTAssertEqual(bestMatches("a", ["a", "c"]), ["a"]) + + XCTAssertEqual(bestMatches("aa", ["aa", "baa", "aab", "aba", "cAa"]), ["aa", "aab", "cAa"]) + + XCTAssertEqual( + bestMatches("resetDownload", ["resetDocumentDownload", "resetZzzDownload", "zrzezszeztzDzozwznzlzozazdz"]), + ["resetZzzDownload", "resetDocumentDownload"] + ) + + XCTAssertEqual(bestMatches("method", ["methodA", "mexthxodB"]), ["methodA"]) + XCTAssertEqual( + bestMatches( + "method", + [ + "methodA", + "methodBSuffixed", + "prefixedMethodB", + "prefixedMethodBSuffixed", + "bigFunction(arg1:arg2:arg3:methodB:arg5:arg6:)", + ] + ), + [ + "methodA", + "methodBSuffixed", + "prefixedMethodB", + "prefixedMethodBSuffixed", + "bigFunction(arg1:arg2:arg3:methodB:arg5:arg6:)", + ] + ) + + XCTAssertEqual( + bestMatches( + "toolTi", + [ + "toolTip", + "removeToolTip(_:)", + "addToolTip(_:)", + ] + ), + [ + "toolTip", + "addToolTip(_:)", + "removeToolTip(_:)", + ] + ) + XCTAssertEqual( + bestMatches( + "UINavigationBar", + [ + SemanticScoredText(1.0, "UINavigationBar", contentType: .projectSymbol), + SemanticScoredText(1.0, "UINavigationBar.m", contentType: .fileName), + SemanticScoredText(1.0, "UINavigationBarBackButtonView", contentType: .projectSymbol), + ] + ), + [ + "UINavigationBar", + "UINavigationBar.m", + "UINavigationBarBackButtonView", + ] + ) + } + + func testCuttingOfMnemonicMatches() { + let badCandidate = "mutableArrayValue(forKey:)" + let goodCandidates = [ + "makeKey()", + "makeMain()", + "makeFirstResponder(:)", + "makeTouchbar()", + "makeBaseWritingDirectionNatural()", + "makeTextWritingDirectionNatural(:)", + ] + let results = bestMatches("mak", goodCandidates + [badCandidate]) + XCTAssertFalse(results.contains(badCandidate)) + XCTAssertEqual(results.count, goodCandidates.count) + } + + func testDontSuggestArgumentOnlyMatches() { + let badSemanticScore = SemanticClassification.allSymbolsClassification.score + let awefulMatches = [ + SemanticScoredText( + "frameSize(forContentSize:horizontalScrollerClass:verticalScrollerClass:borderType:controlSize:scrollerStyle:)", + badSemanticScore + ), + SemanticScoredText( + "contentSize(forFrameSize:horizontalScrollerClass:verticalScrollerClass:borderType:controlSize:scrollerStyle:)", + badSemanticScore + ), + SemanticScoredText( + "init(ri_uuid:ri_user_time:ri_system_time:ri_pkg_idle_wkups:ri_interrupt_wkups:ri_pageins:ri_wired_size:ri_resident_size:ri_phys_footprint:ri_proc_start_abstime:ri_proc_exit_abstime:ri_child_user_time:, ri_child_system_time:ri_child_pkg_idle_wkups:ri_child_interrupt_wkups:ri_child_pageins:ri_child_elapsed_abstime:)", + badSemanticScore + ), + SemanticScoredText( + "init(cmd:cmdsize:rebase_off:rebase_size:bind_off:bind_size:weak_bind_off:weak_bind_size:lazy_bind_off:lazy_bind_size:export_off:export_size:)", + badSemanticScore + ), + SemanticScoredText("init(nTracks:nSizes:sizeTableOffset:trakTable:)", badSemanticScore), + SemanticScoredText( + "init(alertBody:alertLocalizationKey:alertLocalizationArgs:title:titleLocalizationKey:titleLocalizationArgs:subtitle:subtitleLocalizationKey:subtitleLocalizationArgs:alertActionLocalizationKey:alertLaunchImage:soundName:desiredKeys:shouldBadge:shouldSendContentAvailable:shouldSendMutableContent:category:collapseIDKey:)", + badSemanticScore + ), + ] + XCTAssertEqual(bestMatches("resizeable", awefulMatches), []) + } + + func testPrioritizeUserSymbolsOverSDK() { + let projectMethod = SemanticClassification.partial( + completionKind: .function, + moduleProximity: .imported(distance: 0), + scopeProximity: .global + ).score + let frameworkMethod = SemanticClassification.partial( + completionKind: .function, + moduleProximity: .imported(distance: 1), + scopeProximity: .global + ).score + let frameworkType = SemanticClassification.partial( + completionKind: .type, + moduleProximity: .imported(distance: 1), + scopeProximity: .global + ).score + test( + "canceldownload", + precision: .thorough, + prefers: SemanticScoredText(projectMethod, "cancelDownload"), + over: SemanticScoredText(frameworkMethod, "canCancelDownload") + ) + test( + "canceldownload", + precision: .thorough, + prefers: SemanticScoredText(projectMethod, "cancelDownload"), + over: SemanticScoredText(frameworkMethod, "canCancelDownload(_:)") + ) + test( + "canceldownload", + precision: .thorough, + prefers: SemanticScoredText(projectMethod, "cancelDownload"), + over: SemanticScoredText(frameworkMethod, "setCanCancelDownload(_:)") + ) + test( + "canceldownload", + precision: .thorough, + prefers: SemanticScoredText(projectMethod, "cancelDownload"), + over: SemanticScoredText(frameworkMethod, "setCancelDownloadURL(_:)") + ) + test( + "canceldownload", + precision: .thorough, + prefers: SemanticScoredText(projectMethod, "cancelDownload"), + over: SemanticScoredText(frameworkType, "SSXPCMessageCancelDownloads") + ) + test( + "canceldownload", + precision: .thorough, + prefers: SemanticScoredText(projectMethod, "cancelDownload"), + over: SemanticScoredText(frameworkType, "__Reply__CancelDownloadingIconForDisplayIdentrifier_t") + ) + } + + func test33239581() { + let method = SemanticScoredText("testMessagesApp", .partial(completionKind: .function)) + let type = SemanticScoredText( + "XCBuildStepSpecification_CopyMessagesApplicationStub", + .partial(completionKind: .type) + ) + test("testMessagesApp", precision: .thorough, prefers: method, over: type) + } + + func test48642189() { + let method = SemanticScoredText("bestPlaybackRect", .partial(completionKind: .function)) + let enumCase = SemanticScoredText("ISBasePlayerStatusReadyForPlayback", .partial(completionKind: .enumCase)) + test("bestplaybac", precision: .thorough, prefers: method, over: enumCase) + } + + func test24554861_allowMatchingFileExtensionDuringFileNameAcronymMatch() { + XCTAssertEqual( + bestMatches( + "SFD.cpp", + [ + SemanticScoredText(1.0, "SymbolFileDWARF.cpp", contentType: .fileName), + SemanticScoredText(1.0, "MassFileDrop.cpp", contentType: .fileName), + ] + ), + ["SymbolFileDWARF.cpp"] + ) + } + + func test80916856_managedObjectContext() { + let expected = SemanticScoredText( + "NSManagedObjectContext", + .partial( + completionKind: .function, + moduleProximity: .imported(distance: 1), + popularity: .none, + scopeProximity: .global, + structuralProximity: .inapplicable, + synchronicityCompatibility: .compatible, + typeCompatibility: .inapplicable + ) + ) + let decoys = [ + SemanticScoredText( + "MNT_LOCAL", + .partial( + completionKind: .variable, + moduleProximity: .imported(distance: 1), + popularity: .none, + scopeProximity: .global, + structuralProximity: .sdk, + synchronicityCompatibility: .compatible, + typeCompatibility: .inapplicable + ) + ), + SemanticScoredText( + "MM_NOCON", + .partial( + completionKind: .variable, + moduleProximity: .imported(distance: 1), + popularity: .none, + scopeProximity: .global, + structuralProximity: .sdk, + synchronicityCompatibility: .compatible, + typeCompatibility: .inapplicable + ) + ), + ] + XCTAssertEqual(bestMatches("moc", decoys + [expected]).first, expected.text) + } + + func test80916856_translatesAutoresizingMaskIntoConstraints() { + let expected = SemanticScoredText( + "translatesAutoresizingMaskIntoConstraints", + .partial( + completionKind: .function, + moduleProximity: .imported(distance: 1), + popularity: .none, + scopeProximity: .global, + structuralProximity: .sdk, + synchronicityCompatibility: .compatible, + typeCompatibility: .inapplicable + ) + ) + let decoys = [ + SemanticScoredText( + "MTLDynamicLibrary", + .partial( + completionKind: .type, + moduleProximity: .imported(distance: 2), + popularity: .none, + scopeProximity: .global, + structuralProximity: .sdk, + synchronicityCompatibility: .compatible, + typeCompatibility: .inapplicable + ) + ), + SemanticScoredText( + "NSDataWritingAtomic", + .partial( + completionKind: .enumCase, + moduleProximity: .imported(distance: 1), + popularity: .none, + scopeProximity: .global, + structuralProximity: .sdk, + synchronicityCompatibility: .compatible, + typeCompatibility: .inapplicable + ) + ), + ] + XCTAssertEqual(bestMatches("tamic", decoys + [expected]).first, expected.text) + // The repeating pattern at the front exhausts the .thorough scoring search. Even when that happens, we need to + // still find the anchored first character of each leading token match, since it's a special case of fast + // matching. + test( + "tamic", + precision: .thorough, + prefers: "ttttttttttaaaaaaaaaammmmmmmmmmiiiiiiiiiiccccccccccAutoresizingMaskIntoConstraints", + over: "NSDataWritingAtomic" + ) + + } + + func test30956224() { + let type = SemanticScoredText("RunContextManager", .partial(completionKind: .type)) + let method = SemanticScoredText("runContextManager", .partial(completionKind: .function)) + + test("RunContext", precision: .thorough, prefers: type, over: method) + test("runContext", precision: .thorough, prefers: method, over: type) + } + + func test31615625() { + let classFromProject = SemanticScoredText( + "SwiftUnarchiver", + .partial(completionKind: .type, moduleProximity: .imported(distance: 0)) + ) + + let enumFromProject = SemanticScoredText( + "UnarchivingError", + .partial(completionKind: .enumCase, moduleProximity: .imported(distance: 1)) + ) + let classFromSDK = SemanticScoredText( + "NSUnarchiver", + .partial(completionKind: .type, moduleProximity: .imported(distance: 1)) + ) + let typedefFromSDK = SemanticScoredText( + "KeyedUnarchiver", + .partial(completionKind: .type, moduleProximity: .imported(distance: 1)) + ) + + test("Unarchiver", precision: .thorough, prefers: classFromProject, over: enumFromProject) + test("Unarchiver", precision: .thorough, prefers: classFromProject, over: classFromSDK) + test("Unarchiver", precision: .thorough, prefers: classFromProject, over: typedefFromSDK) + } + + func test76104403() throws { + let button = SemanticScoredText( + "Button", + .partial( + availability: .available, + completionKind: .type, + flair: [], + moduleProximity: .imported(distance: 1), + popularity: .none, + scopeProximity: .global, + structuralProximity: .sdk, + synchronicityCompatibility: .unknown, + typeCompatibility: .inapplicable + ) + ) + let menuButtonStyle = SemanticScoredText( + "menuButtonStyle()", + .partial( + availability: .available, + completionKind: .function, + flair: [], + moduleProximity: .imported(distance: 1), + popularity: .none, + scopeProximity: .inheritedContainer, + structuralProximity: .sdk, + synchronicityCompatibility: .unknown, + typeCompatibility: .compatible + ) + ) + let fullPrefix = "Button" + for prefixLength in 1...fullPrefix.count { + let prefix = String(fullPrefix.prefix(prefixLength)) + test(prefix, precision: .thorough, prefers: button, over: menuButtonStyle) + } + } + + func test75842586() { + let goodMatch = SemanticScoredText( + "methodOnString()", + .partial( + completionKind: .function, + flair: [], + moduleProximity: .imported(distance: 0), + scopeProximity: .global + ) + ) + let badMatch = SemanticScoredText( + "methodOnText()", + .partial( + availability: .unknown, + completionKind: .function, + flair: [], + moduleProximity: .unknown, + popularity: .none, + scopeProximity: .unknown, + structuralProximity: .unknown, + synchronicityCompatibility: .unknown, + typeCompatibility: .unknown + ) + ) + XCTAssertEqual(bestMatches("method", [goodMatch, badMatch]), [goodMatch.text]) + } + + func testBestMatchesSemanticCutoffs() { + let typicalScore = SemanticClassification( + availability: .available, + completionKind: .function, + flair: [], + moduleProximity: .same, + popularity: .none, + scopeProximity: .inheritedContainer, + structuralProximity: .sdk, + synchronicityCompatibility: .compatible, + typeCompatibility: .unrelated + ).score + let badScore = SemanticClassification( + availability: .unknown, + completionKind: .function, + flair: [], + moduleProximity: .unknown, + popularity: .unspecified, + scopeProximity: .unknown, + structuralProximity: .unknown, + synchronicityCompatibility: .unknown, + typeCompatibility: .unknown + ).score + XCTAssertEqual( + bestMatches( + "method", + [ + SemanticScoredText("methodA", typicalScore), + SemanticScoredText("methodB", badScore), + ] + ), + ["methodA"] + ) + XCTAssertEqual( + bestMatches( + "method", + [ + SemanticScoredText("mexthxodA", typicalScore), + SemanticScoredText("methodB", badScore), + ] + ), + ["methodB"] + ) + } + + func test75074752() { + XCTAssertEqual( + bestMatches( + "superduper", + [ + "CMAudioSampleBufferCreateWithPacketDescriptions(allocator:dataBuffer:dataReady:makeDataReadyCallback:refcon:formatDescription:sampleCount:presentationTimeStamp:packetDescriptions:sampleBufferOut:)" + ] + ), + [] + ) + XCTAssertEqual( + bestMatches( + "zhskdyd", + [ + "init(calendar:timeZone:era:year:month:day:hour:minute:second:nanosecond:weekday:weekdayOrdinal:quarter:weekOfMonth:weekOfYear:yearForWeekOfYear:)" + ] + ), + [] + ) + XCTAssertEqual( + bestMatches( + "CMAudioSample", + [ + "semanticScore(completionKind:deprecationStatus:flair:moduleProximity:popularity:scopeProximity:structuralProximity:synchronicityCompatibility:typeCompatibility:)" + ] + ), + [] + ) + } + + func test76074456() throws { + let greatTextMatches = [ + SemanticScoredText(0.15, "frame"), + SemanticScoredText(0.15, "frameIdle"), + SemanticScoredText(0.15, "frameRate"), + SemanticScoredText(0.15, "frameBlank"), + SemanticScoredText(0.15, "frameCount"), + SemanticScoredText(0.15, "frameLength"), + SemanticScoredText(0.15, "FrameStyle"), + SemanticScoredText(0.15, "frameComplete"), + SemanticScoredText(0.15, "frameInterval"), + SemanticScoredText(0.15, "frameRotation"), + SemanticScoredText(0.15, "frameDescriptor"), + SemanticScoredText(0.15, "frame(ofColumn:)"), + SemanticScoredText(0.15, "frameAutosaveName"), + SemanticScoredText(0.15, "frameForItem(at:)"), + SemanticScoredText(0.15, "frameCenterRotation"), + SemanticScoredText(0.15, "frame(ofRow:inColumn:)"), + SemanticScoredText(0.15, "frame(withWidth:using:)"), + SemanticScoredText(0.15, "frame(forAlignmentRect:)"), + SemanticScoredText(0.15, "frame(ofInsideOfColumn:)"), + SemanticScoredText(0.15, "frameDidChangeNotification"), + ] + let greatSemanticMatch = SemanticScoredText(1.45, "text(inFrames:)") + XCTAssertEqual( + bestMatches("frame", greatTextMatches + [greatSemanticMatch], maximumNumberOfItemsForExpensiveSelection: 5) + .first, + greatSemanticMatch.text + ) + } + + func testMatchCollatorUsesThoroughScoringWithOneCharacterPatternIfThereAreFewMatches_100039832() { + let pattern = Pattern(text: "h") + // During fast matching, these will both match like "widt[h]:...", but during the thorough match one will match + // like "width:[h]eight:alignment:". We want to ensure that's happening even though the filter text isn't as + // long as `MatchCollator.minimumPatternLengthToAlwaysRescoreWithThoroughPrecision`. + let batch = CandidateBatch(symbols: [ + "width:alignment:", + "width:height:alignment:", + ]) + let matches = batch.indices.map { index in + MatchCollator.Match( + batchIndex: 0, + candidateIndex: index, + groupID: nil, + score: .init(textComponent: 1.0, semanticComponent: 1.0) + ) + } + let selection = MatchCollator.selectBestMatches( + matches, + from: [batch], + for: pattern, + influencingTokenizedIdentifiers: [], + orderingTiesBy: { _, _ in false }, + maximumNumberOfItemsForExpensiveSelection: MatchCollator.defaultMaximumNumberOfItemsForExpensiveSelection + ) + let bestMatches = selection.matches.map { match in + batch[stringAt: match.candidateIndex] + } + XCTAssertEqual(bestMatches.first, "width:height:alignment:") + XCTAssertEqual(selection.precision, .thorough) + } + + func testBestMatchesBoundaries() { + for patternLength in 0..<(MatchCollator.minimumPatternLengthToAlwaysRescoreWithThoroughPrecision + 1) { + let patternText = String(repeating: "a", count: patternLength) + let differingCandidates = (0..<(MatchCollator.defaultMaximumNumberOfItemsForExpensiveSelection * 2)).map { + length in + patternText + String(repeating: "a", count: length) + } + let justTooFewIdenticalCandidates = Array( + repeating: patternText, + count: MatchCollator.defaultMaximumNumberOfItemsForExpensiveSelection - 1 + ) + let justEnoughIdenticalCandidates = Array( + repeating: patternText, + count: MatchCollator.defaultMaximumNumberOfItemsForExpensiveSelection + ) + let justTooManyIdenticalCandidates = Array( + repeating: patternText, + count: MatchCollator.defaultMaximumNumberOfItemsForExpensiveSelection + 1 + ) + for candidates in [ + justTooFewIdenticalCandidates, justEnoughIdenticalCandidates, justTooManyIdenticalCandidates, + differingCandidates, + ] { + let matches = bestMatches(patternText, candidates) + if patternLength < MatchCollator.minimumPatternLengthToAlwaysRescoreWithThoroughPrecision { + XCTAssertEqual(matches, candidates) + } else { + let expectedMax = MatchCollator.defaultMaximumNumberOfItemsForExpensiveSelection + XCTAssert( + matches == Array(candidates.prefix(expectedMax)), + "Expected \(expectedMax) matches, but got:\n - \(matches.joined(separator: "\n - "))" + ) + } + } + } + } + + private struct CompletionExample { + var patternUTF8Length: Int + var textScore: Double + var pattern: String + var candidate: String + init(pattern: String, candidate: String) { + self.pattern = pattern + self.candidate = candidate + self.patternUTF8Length = pattern.utf8.count + self.textScore = self.candidate.withUTF8 { candidateUTF8Bytes in + Pattern(text: pattern).score( + candidate: candidateUTF8Bytes, + contentType: .codeCompletionSymbol, + precision: .thorough + ) + } + } + } + + private var badCompletionExamples: [CompletionExample] = { + return [ + ("po", "exponent"), + ("on", "exponent"), + ("ne", "exponent"), + ("nf", "infinity"), + ("fi", "infinity"), + ("ni", "infinity"), + ("ghi", "rightMouseUp(with:)"), + ("oef", "logReferenceTree()"), + ("oef", "makeBaseWritingDirectionLeftToRight()"), + ("cron", "scrollRectToVisible(,animated:)"), + ("ptop", "newScriptingObject(of:,forValueForKey:,withContentsValue:,properties:)"), + ("ptoc", "init(format:,options:,locale:)"), + ("abcde", "outlineTableColumnIndex"), + ("aeiou", "classForUserDefinedRuntimeAttributesPlaceholder()"), + ("croner", "descriptionForAssertionMessage"), + ("cornerz", "init(recordZonesToSave:,recordZoneIDsToDelete:)"), + ("holferf", "shouldBufferInAnticipationOfPlayback"), + ( + "ounasdaa", + "init(chunkSampleCount:,chunkHasUniformSampleSizes:,chunkHasUniformSampleDurations:,chunkHasUniformFormatDescriptions:)" + ), + ("ounasdaal", "init(nickname:,number:,accountType:,organizationName:,balance:,secondaryBalance:)"), + ("ounasdaar", "init(time:,type:,isDurationBoundary:,isMarkerBoundary:,isSelectedTimeRangeBoundary:)"), + ( + "ounasduari", + "init(chunkSampleCount:,chunkHasUniformSampleSizes:,chunkHasUniformSampleDurations:,chunkHasUniformFormatDescriptions:)" + ), + ].map(CompletionExample.init) + }() + + /// A longer match has a higher score, so matching "a" against "aaa" might score 1, and then matching "aa" against + /// "aaa" might score as 2. Because of this, the cutoff is a function of the length. + private func generateBestRejectedTextScoreByPatternLength() -> [Double] { + let maxPatternUTF8Length = badCompletionExamples.map(\.patternUTF8Length).max() ?? 0 + var bestRejectedTextScoreByPatternLength: [Double] = Array(repeating: 0, count: maxPatternUTF8Length + 1) + + for badCompletionExample in badCompletionExamples { + bestRejectedTextScoreByPatternLength[badCompletionExample.patternUTF8Length] = max( + bestRejectedTextScoreByPatternLength[badCompletionExample.patternUTF8Length], + badCompletionExample.textScore + ) + } + + // If we computed a cutoff of 1.5 for a 4 character pattern, and then our worst example for a 5 character + // pattern was 1.1, then, use the 1.5 value since longer matches have longer scores. + for (previousIndex, nextIndex) in zip( + bestRejectedTextScoreByPatternLength.indices, + bestRejectedTextScoreByPatternLength.indices.dropFirst() + ) { + bestRejectedTextScoreByPatternLength[nextIndex] = max( + bestRejectedTextScoreByPatternLength[nextIndex], + bestRejectedTextScoreByPatternLength[previousIndex] + ) + } + + return bestRejectedTextScoreByPatternLength + } + + func testMinimumTextCutoff() { + for awfulCompletionExample in badCompletionExamples { + let matches = self.bestMatches(awfulCompletionExample.pattern, [awfulCompletionExample.candidate]) + XCTAssertEqual(matches, []) + } + if MatchCollator.bestRejectedTextScoreByPatternLength != generateBestRejectedTextScoreByPatternLength() { + let literals = generateBestRejectedTextScoreByPatternLength().map { cutoff in + " \(cutoff),\n" + }.joined() + let code = + " internal static let bestRejectedTextScoreByPatternLength: [Double] = [\n" + literals + " ]\n" + XCTFail("Update MatchCollator.bestRejectedTextScoreByPatternLength to:\n" + code) + } + } + + func testBorderlineCompletionsThatShouldPassMinimumTextCutoff() { + for leadingDecoy in [ + "", "zzz", "zzzZzzzzz", "zzzZzzzzzZzzzzzzzzzzz", "zzzZzzzzzZzzzzzzzzzzzZzzzzzzzzzzzzzzzzzzzzzzz", + ] { + for trailingDecoy in [ + "", "Zzz", "ZzzZzzzzz", "ZzzZzzzzzZzzzzzzzzzzz", "ZzzZzzzzzZzzzzzzzzzzzZzzzzzzzzzzzzzzzzzzzzzzz", + ] { + func test(_ pattern: String, matches candidate: String) { + let casedCandidate = leadingDecoy.isEmpty ? candidate : candidate.capitalized + let disguised = leadingDecoy + casedCandidate + trailingDecoy + let matches = bestMatches(pattern, [disguised]) + XCTAssertEqual(matches, [disguised]) + } + test("order", matches: "border") + test("load", matches: "download") + test("not", matches: "cannot") + test("our", matches: "your") + test("ous", matches: "supercalifragilisticexpialidocious") + test("frag", matches: "supercalifragilisticexpialidocious") + } + } + } + + func testDecentMatchesAreNotCompletelyExcludedGivenGreatMatches() { + func test(_ filter: String, _ candidates: [String]) { + let matches = bestMatches(filter, candidates) + XCTAssertEqual(Set(matches), Set(candidates)) + } + test("GKPlayer", ["GKPlayer", "GKLocalPlayer"]) + test( + "MTLShar", + ["MTLSharedEventHandle", "MTLSharedEventListener", "MTLSharedTextureHandle", "MTLStorageModeShared"] + ) + } + + func testExclusionOfMiddleOfWordCompactMatches() { + XCTAssertEqual(bestMatches("HSta", ["HStack", "ABC_FHSTAT"]), ["HStack"]) + XCTAssertEqual(bestMatches("HSta", ["HStack", "errSecCSHostProtocolStateError"]), ["HStack"]) + } + + func testTypeNameOverLocalWithCaseMatch_88653597() { + let localVariable = SemanticScoredText( + "imageCache", + SemanticClassification( + availability: .available, + completionKind: .variable, + flair: [], + moduleProximity: .imported(distance: 0), + popularity: .none, + scopeProximity: .local, + structuralProximity: .project(fileSystemHops: 0), + synchronicityCompatibility: .compatible, + typeCompatibility: .compatible + ) + ) + let typeName = SemanticScoredText( + "ImageCache", + SemanticClassification( + availability: .available, + completionKind: .type, + flair: [], + moduleProximity: .imported(distance: 0), + popularity: .none, + scopeProximity: .global, + structuralProximity: .project(fileSystemHops: 0), + synchronicityCompatibility: .compatible, + typeCompatibility: .compatible + ) + ) + for prefixLength in 1...typeName.text.count { + let prefix = String(typeName.text.prefix(prefixLength)) + XCTAssertEqual(bestMatches(prefix, [localVariable, typeName]), [typeName.text, localVariable.text]) + } + } + + func test75845907() { + let patternText = "separateSecondary" + patternText.enumeratePrefixes(includeLowercased: true) { patternText in + let results = bestMatches( + patternText, + [ + "separateSecondaryViewController(for:)", + "splitViewController(:separateSecondaryFrom:)", + ] + ) + XCTAssertEqual( + results.count, + 2, + "The contiguous pattern \"\(patternText)\" should still show both results." + ) + } + } + + func testOrderingOverloads() { + struct Completion { + var filterText: String + var displayText: String + } + let completions: [Completion] = [ + .init(filterText: "theFunction(argument:)", displayText: "theFunction(argument: Int)"), + .init(filterText: "theFunction(argument:)", displayText: "theFunction(argument: String)"), + .init(filterText: "theFunction(argument:)", displayText: "theFunction(argument: Double)"), + .init(filterText: "theFunction(argument:)", displayText: "theFunction(argument: Bool)"), + .init(filterText: "theFunction(argument:)", displayText: "theFunction(argument: UInt)"), + ] + let pattern = Pattern(text: "the") + let batch = CandidateBatch(symbols: completions.map(\.filterText)) + let textScoredMatches = pattern.scoredMatches(across: [batch], precision: .fast) + let matches: [MatchCollator.Match] = textScoredMatches.map { match in + MatchCollator.Match( + batchIndex: match.batchIndex, + candidateIndex: match.candidateIndex, + groupID: nil, + score: CompletionScore(textComponent: match.textScore, semanticComponent: 1.0) + ) + } + let bestMatches = MatchCollator.selectBestMatches( + for: pattern, + from: matches, + in: [batch], + influencingTokenizedIdentifiers: [] + ) { lhs, rhs in + let lhsDisplayText = completions[lhs.candidateIndex].displayText + let rhsDisplayText = completions[rhs.candidateIndex].displayText + return lhsDisplayText < rhsDisplayText + } + let bestMatchesDisplayText = bestMatches.map { bestMatch in + completions[bestMatch.candidateIndex].displayText + } + XCTAssertEqual( + bestMatchesDisplayText, + [ + "theFunction(argument: Bool)", + "theFunction(argument: Double)", + "theFunction(argument: Int)", + "theFunction(argument: String)", + "theFunction(argument: UInt)", + ] + ) + } + + func testDeprecatedGroupMembersComeLater() { + // Notice that the semantically worse results are textually better matches - they're shorter, and sort + // lexicographically earlier. + let type = "TextField" + let initializer = "TextField(:text:prompt:)" + let softDeprecatedInitializer = "TextField(:text:)" + let deprecatedInitializer = "TextField()" + let items = [ + SemanticScoredText(type, SemanticClassification.partial(completionKind: .type).score, groupID: 1), + SemanticScoredText( + initializer, + SemanticClassification.partial(completionKind: .initializer).score, + groupID: 1 + ), + SemanticScoredText( + softDeprecatedInitializer, + SemanticClassification.partial(availability: .softDeprecated, completionKind: .initializer).score, + groupID: 1 + ), + SemanticScoredText( + deprecatedInitializer, + SemanticClassification.partial(availability: .deprecated, completionKind: .initializer).score, + groupID: 1 + ), + ] + XCTAssertEqual( + bestMatches("TextField", items), + [type, initializer, deprecatedInitializer, softDeprecatedInitializer] + ) + } + + func testOverlappingAndSparseGroupIDs() { + func scoreAsTiesAndSelectAll( + pattern: Pattern, + overlappingGroupID: Int, + from batchesOfScoredText: [[SemanticScoredText]] + ) -> [String] { + typealias Match = MatchCollator.Match + let batches = batchesOfScoredText.map { batchOfScoredText in + CandidateBatch(candidates: batchOfScoredText) + } + var matches: [Match] = [] + for (batchIndex, batch) in batchesOfScoredText.enumerated() { + for (candidateIndex, semanticScoredText) in batch.enumerated() { + let score = CompletionScore(textComponent: 1, semanticComponent: semanticScoredText.semanticScore) + matches.append( + Match( + batchIndex: batchIndex, + candidateIndex: candidateIndex, + groupID: overlappingGroupID, + score: score + ) + ) + } + } + let bestMatches = MatchCollator.selectBestMatches( + for: pattern, + from: matches, + in: batches, + influencingTokenizedIdentifiers: [], + orderingTiesBy: { _, _ in false } + ) + return bestMatches.map { match in + batches[match.batchIndex][stringAt: match.candidateIndex] + } + } + + for groupID in [0, 1, -1, Int.max, Int.min] { + for reverseBatches in [false, true] { + for reverseA in [false, true] { + for reverseB in [false, true] { + for aSemanticScore in [1.0, 2.0] { + for bSemanticScore in [1.0, 2.0] { + let aText = ["pA", "pA()", "pA(a:)"] + let bText = ["pB", "pB()", "pB(b:)"] + let aScoredText = aText.map { SemanticScoredText($0, aSemanticScore) } + let bScoredResults = bText.map { SemanticScoredText($0, bSemanticScore) } + let results = scoreAsTiesAndSelectAll( + pattern: Pattern(text: "p"), + overlappingGroupID: groupID, + from: [ + aScoredText.conditionallyReversed(reverseA), + bScoredResults.conditionallyReversed(reverseB), + ].conditionallyReversed(reverseBatches) + ) + let expectedOrder = aSemanticScore >= bSemanticScore ? aText + bText : bText + aText + XCTAssertEqual(results, expectedOrder) + } + } + } + } + } + } + } + + func testGrouping() { + let okTypeScore = SemanticClassification( + availability: .available, + completionKind: .type, + flair: [], + moduleProximity: .imported(distance: 0), + popularity: .none, + scopeProximity: .global, + structuralProximity: .sdk, + synchronicityCompatibility: .compatible, + typeCompatibility: .inapplicable + ).score + let okInitializerScore = SemanticClassification( + availability: .available, + completionKind: .initializer, + flair: [], + moduleProximity: .imported(distance: 0), + popularity: .none, + scopeProximity: .global, + structuralProximity: .sdk, + synchronicityCompatibility: .compatible, + typeCompatibility: .inapplicable + ).score + + let greatTypeScore = SemanticClassification( + availability: .available, + completionKind: .type, + flair: [], + moduleProximity: .same, + popularity: .none, + scopeProximity: .global, + structuralProximity: .project(fileSystemHops: 0), + synchronicityCompatibility: .compatible, + typeCompatibility: .compatible + ).score + let greatInitializerScore = SemanticClassification( + availability: .available, + completionKind: .initializer, + flair: [], + moduleProximity: .same, + popularity: .none, + scopeProximity: .global, + structuralProximity: .project(fileSystemHops: 0), + synchronicityCompatibility: .compatible, + typeCompatibility: .compatible + ).score + + let greatLocalScore = SemanticClassification( + availability: .available, + completionKind: .variable, + flair: [], + moduleProximity: .same, + popularity: .none, + scopeProximity: .local, + structuralProximity: .project(fileSystemHops: 0), + synchronicityCompatibility: .compatible, + typeCompatibility: .compatible + ).score + let poorGlobalScore = SemanticClassification( + availability: .available, + completionKind: .initializer, + flair: [], + moduleProximity: .imported(distance: 1), + popularity: .none, + scopeProximity: .global, + structuralProximity: .sdk, + synchronicityCompatibility: .compatible, + typeCompatibility: .compatible + ).score + + let results = bestMatches( + "tex", + [ + SemanticScoredText("text", greatLocalScore), + SemanticScoredText("initializeTextSubsystem()", poorGlobalScore), + SemanticScoredText("Text", okTypeScore, groupID: 0), + SemanticScoredText("TextualAnalyzer", okTypeScore, groupID: 2), + SemanticScoredText("TextField", greatTypeScore, groupID: 1), + SemanticScoredText("Text(string:)", okInitializerScore, groupID: 0), + SemanticScoredText("Text(string:encoding:conicalize:validate:)", okInitializerScore, groupID: 0), + SemanticScoredText("TextField()", greatInitializerScore, groupID: 1), + SemanticScoredText("TextField(text:)", greatInitializerScore, groupID: 1), + SemanticScoredText( + "TextField(text:alignment:wrapping:maximumNumberOfLines:font:)", + okInitializerScore, + groupID: 1 + ), + SemanticScoredText("TextualAnalyzer()", okInitializerScore, groupID: 2), + ] + ) + XCTAssertEqual( + results, + [ + "text", + + "TextField", + "TextField()", + "TextField(text:)", + "TextField(text:alignment:wrapping:maximumNumberOfLines:font:)", + + "Text", + "Text(string:)", + "Text(string:encoding:conicalize:validate:)", + + "TextualAnalyzer", + "TextualAnalyzer()", + + "initializeTextSubsystem()", + ] + ) + + } + + func testGroupingWithEqualGroupScores() { + for (foodID, footID) in [(0, 1), (1, 0)] { + let candidates = [ + SemanticScoredText("food", groupID: foodID), + SemanticScoredText("foot", groupID: footID), + SemanticScoredText("food()", groupID: foodID), + SemanticScoredText("foot()", groupID: footID), + ] + let expectedOrder = [ + "food", + "food()", + "foot", + "foot()", + ] + let prefix = "foo" + XCTAssertEqual(bestMatches(prefix, candidates), expectedOrder) + XCTAssertEqual(bestMatches(prefix, candidates.reversed()), expectedOrder) // Verify initial order doesn't matter + } + } + + func testBulkLoading() { + typealias UTF8Bytes = Pattern.UTF8Bytes + let typeStrings = [ + "", + "a", + "Word", + ] + let typeUTF8Buffers = typeStrings.map { typeString in + typeString.allocateCopyOfUTF8Buffer() + }; + defer { + for typeUTF8Buffer in typeUTF8Buffers { + typeUTF8Buffer.deallocate() + } + } + + let bulkStringLoaded = CandidateBatch(candidates: typeStrings, contentType: .unknown) + let bulkByteLoaded = CandidateBatch(candidates: typeUTF8Buffers, contentType: .unknown) + let singeBytesLoaded = { // To get unused variables if we forget to compare one. + var batch = CandidateBatch() + for typeUTF8Buffer in typeUTF8Buffers { + batch.append(typeUTF8Buffer, contentType: .unknown) + } + return batch + }() + + let singeStringLoaded = { + var batch = CandidateBatch() + for typeString in typeStrings { + batch.append(typeString, contentType: .unknown) + } + return batch + }() + + let baseline = bulkStringLoaded + XCTAssertEqual(baseline, bulkStringLoaded) + XCTAssertEqual(baseline, bulkByteLoaded) + XCTAssertEqual(baseline, singeBytesLoaded) + XCTAssertEqual(baseline, singeStringLoaded) + } +} + +fileprivate extension CandidateBatch { + var strings: [String] { + (0..(pattern: Pattern, precision: Pattern.Precision, expression: (Int, Double) -> T) -> [T] { + var resutls: [T] = [] + enumerate { candidateIndex, candidate in + if pattern.matches(candidate: candidate) { + resutls.append(expression(candidateIndex, pattern.score(candidate: candidate, precision: precision))) + } + } + return resutls + } +} + +fileprivate extension Pattern { + func seariallyScoreMatches(in batch: CandidateBatch, precision: Precision) -> [CandidateBatchMatch] { + let resutls = batch.mapMatches(pattern: self, precision: precision) { index, score in + CandidateBatchMatch(candidateIndex: index, textScore: score) + } + return resutls + } + + func seariallyScoreMatches(across batches: [CandidateBatch], precision: Precision) -> [CandidateBatchesMatch] { + var combinedResults: [CandidateBatchesMatch] = [] + for (batchIndex, batch) in batches.enumerated() { + let batchResults = batch.mapMatches(pattern: self, precision: precision) { candidateIndex, score in + CandidateBatchesMatch(batchIndex: batchIndex, candidateIndex: candidateIndex, textScore: score) + } + combinedResults.append(contentsOf: batchResults) + } + return combinedResults + } +} + +extension Array { + func conditionallyReversed(_ condition: Bool) -> Array { + condition ? reversed() : self + } +} + +extension SemanticClassification { + static let allSymbolsClassification = Self( + availability: .unknown, + completionKind: .unknown, + flair: [], + moduleProximity: .unknown, + popularity: .none, + scopeProximity: .unknown, + structuralProximity: .unknown, + synchronicityCompatibility: .unknown, + typeCompatibility: .unknown + ) +} diff --git a/Tests/CompletionScoringTests/CompletionScoreTests.swift b/Tests/CompletionScoringTests/CompletionScoreTests.swift new file mode 100644 index 000000000..d216abbf3 --- /dev/null +++ b/Tests/CompletionScoringTests/CompletionScoreTests.swift @@ -0,0 +1,277 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import CompletionScoring +import XCTest + +class CompletionScoringTests: XCTestCase { + func testSelfEvidentPriorities() { + func assertGreaterThan(_ lhs: SemanticClassification, _ rhs: SemanticClassification) { + XCTAssertGreaterThan(lhs.score, rhs.score) + } + + func assertGreaterThanOrEqual(_ lhs: SemanticClassification, _ rhs: SemanticClassification) { + XCTAssertGreaterThanOrEqual(lhs.score, rhs.score) + } + + assertGreaterThan( + .partial(completionKind: .variable, scopeProximity: .local), + .partial(completionKind: .variable, scopeProximity: .argument) + ) + assertGreaterThan( + .partial(completionKind: .variable, scopeProximity: .argument), + .partial(completionKind: .variable, scopeProximity: .container) + ) + assertGreaterThan( + .partial(completionKind: .variable, scopeProximity: .container), + .partial(completionKind: .variable, scopeProximity: .inheritedContainer) + ) + assertGreaterThan( + .partial(completionKind: .variable, scopeProximity: .inheritedContainer), + .partial(completionKind: .variable, scopeProximity: .global) + ) + + assertGreaterThan( + .partial(completionKind: .enumCase, scopeProximity: .container), + .partial(completionKind: .variable, scopeProximity: .container) + ) + assertGreaterThan( + .partial(completionKind: .variable, scopeProximity: .container), + .partial(completionKind: .function, scopeProximity: .container) + ) + + assertGreaterThan( + .partial(completionKind: .function, flair: [.chainedCallToSuper], scopeProximity: .inheritedContainer), + .partial(completionKind: .function, scopeProximity: .inheritedContainer) + ) + + assertGreaterThan( + .partial(completionKind: .variable, scopeProximity: .container), + .partial(completionKind: .variable, flair: [.chainedMember], scopeProximity: .container) + ) + + assertGreaterThan( + .partial(completionKind: .function, moduleProximity: .imported(distance: 0)), + .partial(completionKind: .function, moduleProximity: .imported(distance: 1)) + ) + + assertGreaterThan( + .partial(completionKind: .function, moduleProximity: .imported(distance: 1)), + .partial(completionKind: .function, moduleProximity: .importable) + ) + + assertGreaterThan( + .partial(completionKind: .function, structuralProximity: .project(fileSystemHops: 0)), + .partial(completionKind: .function, structuralProximity: .project(fileSystemHops: 1)) + ) + + assertGreaterThan( + .partial(completionKind: .function, synchronicityCompatibility: .compatible), + .partial(completionKind: .function, synchronicityCompatibility: .convertible) + ) + + assertGreaterThan( + .partial(completionKind: .function, synchronicityCompatibility: .convertible), + .partial(completionKind: .function, synchronicityCompatibility: .incompatible) + ) + + assertGreaterThan( + .partial(completionKind: .function, typeCompatibility: .compatible), + .partial(completionKind: .function, typeCompatibility: .unrelated) + ) + + assertGreaterThanOrEqual( + .partial(availability: .available, completionKind: .function), + .partial(availability: .deprecated, completionKind: .function) + ) + + assertGreaterThanOrEqual( + .partial(availability: .deprecated, completionKind: .function), + .partial(availability: .unavailable, completionKind: .function) + ) + + assertGreaterThan( + .partial(completionKind: .function, scopeProximity: .global), + .partial(completionKind: .initializer, scopeProximity: .global) + ) + + assertGreaterThan( + .partial(completionKind: .argumentLabels, scopeProximity: .global), + .partial(completionKind: .function, scopeProximity: .container) + ) + + assertGreaterThan( + .partial(completionKind: .type), + .partial(completionKind: .module) + ) + + assertGreaterThan( + .partial(completionKind: .template), + .partial(completionKind: .variable) + ) + } + + func testSymbolNotoriety() { + let stream: PopularityIndex.Symbol = "Foundation.NSStream" + let string: PopularityIndex.Symbol = "Foundation.NSString" + PopularityIndex(notoriousSymbols: [string]).expect(stream, >, string) + PopularityIndex(notoriousSymbols: [stream]).expect(string, >, stream) + } + + func testTypeReferencePercentages() { + let stream: PopularityIndex.Symbol = "Foundation.NSStream" + let string: PopularityIndex.Symbol = "Foundation.NSString" + PopularityIndex(flatReferencePercentages: [string: 0.25]).expect(string, >, stream) + PopularityIndex(flatReferencePercentages: [stream: 0.25]).expect(stream, >, string) + } + + func testModuleCombinesWithReferenceFrequency() { + let fopen: PopularityIndex.Symbol = "POSIX.fopen" + let array: PopularityIndex.Symbol = "Swift.Array" + PopularityIndex(flatReferencePercentages: [fopen: 0.501, array: 0.500], notoriousModules: ["POSIX"]).expect( + array, + >, + fopen + ) + PopularityIndex(flatReferencePercentages: [fopen: 0.501, array: 0.500], popularModules: ["Swift"]).expect( + array, + >, + fopen + ) + } + + func testMethodReferencePercentages() { + let index = PopularityIndex(referencePercentages: [ + "Swift.Array": [ + "count": 0.75, + "isEmpty": 0.25, + ], + "Swift.String": [ + "isEmpty": 0.75, + "count": 0.25, + ], + ]) + index.expect("Swift.Array.count", >, "Swift.String.count") + index.expect("Swift.String.isEmpty", >, "Swift.Array.isEmpty") + } + + func testModulePopularity() { + let index = PopularityIndex(popularModules: ["Swift"], notoriousModules: ["POSIX"]) + index.expect("Mine.Type", >, "POSIX.Type") + index.expect("Mine.Type", <, "Swift.Type") + index.expect("Mine.Type", ==, "Yours.Type") + } + + func testPopularitySerialization() { + let original = PopularityIndex( + referencePercentages: [ + "Swift.Array": [ + "count": 0.75, + "isEmpty": 0.25, + ], + "Swift.String": [ + "isEmpty": 0.75, + "count": 0.25, + ], + ], + popularModules: ["Swift"], + notoriousModules: ["POSIX"] + ) + XCTAssertNoThrow( + try { + let copy = try PopularityIndex.deserialize(data: original.serialize(version: .initial)) + copy.expect("Swift.Array.count", >, "Swift.array.isEmpty") + copy.expect("Swift.String.isEmpty", >, "Swift.array.count") + copy.expect("Swift.Unremarkable", >, "POSIX.Unremarkable") + XCTAssertEqual(original.symbolPopularity, copy.symbolPopularity) + XCTAssertEqual(original.modulePopularity, copy.modulePopularity) + }() + ) + } +} + +extension PopularityIndex { + func expect(_ lhs: Symbol, _ comparison: (Double, Double) -> Bool, _ rhs: Symbol) { + XCTAssert(comparison(popularity(of: lhs).scoreComponent, popularity(of: rhs).scoreComponent)) + } +} + +extension PopularityIndex.Scope { + public init(stringLiteral value: StringLiteralType) { + self.init(string: value) + } + + public init(string value: StringLiteralType) { + let splits = value.split(separator: ".").map(String.init) + if splits.count == 1 { + self.init(container: nil, module: splits[0]) + } else if splits.count == 2 { + self.init(container: splits[1], module: splits[0]) + } else { + preconditionFailure("Invalid scope \(value)") + } + } +} + +extension PopularityIndex.Scope: ExpressibleByStringLiteral {} + +extension PopularityIndex.Symbol { + public init(stringLiteral value: StringLiteralType) { + self.init(string: value) + } + + public init(string value: StringLiteralType) { + let splits = value.split(separator: ".").map(String.init) + if splits.count == 2 { + self.init(name: splits[1], scope: .init(container: nil, module: splits[0])) + } else if splits.count == 3 { + self.init(name: splits[2], scope: .init(container: splits[1], module: splits[0])) + } else { + preconditionFailure("Invalid symbol name \(value)") + } + } +} +extension PopularityIndex.Symbol: ExpressibleByStringLiteral {} + +extension PopularityIndex { + init( + referencePercentages: [Scope: [String: Double]] = [:], + notoriousSymbols: [Symbol] = [], + popularModules: [String] = [], + notoriousModules: [String] = [] + ) { + self.init( + symbolReferencePercentages: referencePercentages, + notoriousSymbols: notoriousSymbols, + popularModules: popularModules, + notoriousModules: notoriousModules + ) + } + + init( + flatReferencePercentages: [Symbol: Double], + notoriousSymbols: [Symbol] = [], + popularModules: [String] = [], + notoriousModules: [String] = [] + ) { + var symbolReferencePercentages: [Scope: [String: Double]] = [:] + for (symbol, percentage) in flatReferencePercentages { + symbolReferencePercentages[symbol.scope, default: [:]][symbol.name] = percentage + } + self.init( + symbolReferencePercentages: symbolReferencePercentages, + notoriousSymbols: notoriousSymbols, + popularModules: popularModules, + notoriousModules: notoriousModules + ) + } +} diff --git a/Tests/CompletionScoringTests/InfluencingIdentifiersTests.swift b/Tests/CompletionScoringTests/InfluencingIdentifiersTests.swift new file mode 100644 index 000000000..b8109542f --- /dev/null +++ b/Tests/CompletionScoringTests/InfluencingIdentifiersTests.swift @@ -0,0 +1,91 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import CompletionScoring +import XCTest + +class InfluencingIdentifiersTests: XCTestCase { + + private func score(_ candidate: String, given identifiers: [String]) -> Double { + let tokenizedIdentifiers = MatchCollator.tokenize( + influencingTokenizedIdentifiers: identifiers, + filterLowSignalTokens: true + ) + return InfluencingIdentifiers.withUnsafeInfluencingTokenizedIdentifiers(tokenizedIdentifiers) { + influencingIdentifiers in + influencingIdentifiers.score(text: candidate) + } + } + + func testEmptySets() { + // Just test for / 0 type mistakes. + XCTAssertEqual(score("", given: []), 0) + XCTAssertEqual(score("document", given: []), 0) + XCTAssertEqual(score("", given: ["document"]), 0) + } + + private func expect( + _ lhs: String, + _ comparator: (Double, Double) -> Bool, + _ rhs: String, + whenInfluencedBy influencers: [String] + ) { + XCTAssert(comparator(score(lhs, given: influencers), score(rhs, given: influencers))) + } + + func testNonMatches() { + expect("only", >, "decoy", whenInfluencedBy: ["only"]) + expect("decoy", ==, "lure", whenInfluencedBy: ["only"]) + } + + func testIdentifierPositions() { + expect("first", >, "second", whenInfluencedBy: ["first", "second", "third"]) + expect("second", >, "third", whenInfluencedBy: ["first", "second", "third"]) + } + + func testTokenPositions() { + expect("first", ==, "second", whenInfluencedBy: ["firstSecondThird"]) + expect("second", ==, "third", whenInfluencedBy: ["firstSecondThird"]) + } + + func testTokenCoverage() { + expect("first", <, "firstSecond", whenInfluencedBy: ["firstSecondThird"]) + expect("firstSecond", <, "firstSecondThird", whenInfluencedBy: ["firstSecondThird"]) + expect("second", <, "secondThird", whenInfluencedBy: ["firstSecondThird"]) + expect("firstSecond", ==, "secondThird", whenInfluencedBy: ["firstSecondThird"]) + expect("firstThird", ==, "thirdSecond", whenInfluencedBy: ["firstSecondThird"]) + } + + func testCaseInvariance() { + expect("NSWindowController", >, "NSController", whenInfluencedBy: ["windowController"]) + expect("NSWindowController", >, "NSWindow", whenInfluencedBy: ["windowController"]) + } + + func testInertTerms() { + // Too Short + expect("decoy", ==, "to", whenInfluencedBy: ["to"]) + expect("decoy", ==, "URL", whenInfluencedBy: ["URL"]) + // Specifically ignored + expect("decoy", ==, "from", whenInfluencedBy: ["from"]) + expect("decoy", ==, "with", whenInfluencedBy: ["with"]) + } +} + +fileprivate extension InfluencingIdentifiers { + func score(text: String) -> Double { + UnsafeStackAllocator.withUnsafeStackAllocator { allocator in + Candidate.withAccessToCandidate(for: text, contentType: .codeCompletionSymbol) { candidate in + score(candidate: candidate, allocator: &allocator) + } + } + } +} diff --git a/Tests/CompletionScoringTests/PatternTests.swift b/Tests/CompletionScoringTests/PatternTests.swift new file mode 100644 index 000000000..ae6b3be8e --- /dev/null +++ b/Tests/CompletionScoringTests/PatternTests.swift @@ -0,0 +1,1301 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import CompletionScoring +import XCTest + +class PatternTests: XCTestCase { + typealias ContentType = Candidate.ContentType + func testMatches() throws { + func test(pattern patternText: String, candidate candidateText: String, match expectedMatch: Bool) { + let pattern = Pattern(text: patternText) + let actualMatch = Candidate.withAccessToCandidate(for: candidateText, contentType: .codeCompletionSymbol) { + candidate in + pattern.matches(candidate: candidate) + } + XCTAssertEqual(actualMatch, expectedMatch) + } + test(pattern: "", candidate: "", match: true) + test(pattern: "", candidate: "a", match: true) + test(pattern: "a", candidate: "a", match: true) + test(pattern: "a", candidate: "aa", match: true) + test(pattern: "aa", candidate: "a", match: false) + test(pattern: "b", candidate: "a", match: false) + test(pattern: "b", candidate: "ba", match: true) + test(pattern: "b", candidate: "ab", match: true) + test(pattern: "ba", candidate: "a", match: false) + test(pattern: "ba", candidate: "ba", match: true) + test(pattern: "ab", candidate: "ba", match: false) + test(pattern: "aaa", candidate: "aabb", match: false) + } + + func withTokenization( + of text: String, + contentType: Pattern.ContentType, + body: (Pattern.UTF8Bytes, Pattern.Tokenization) throws -> R + ) rethrows -> R { + try UnsafeStackAllocator.withUnsafeStackAllocator { allocator in + try text.withUncachedUTF8Bytes { mixedcaseBytes in + var tokenization = Pattern.Tokenization.allocate( + mixedcaseBytes: mixedcaseBytes, + contentType: contentType, + allocator: &allocator + ); defer { tokenization.deallocate(allocator: &allocator) } + return try body(mixedcaseBytes, tokenization) + } + } + } + + func testTokenization() throws { + func test(text escapedText: String) { + let expectedTokens = escapedText.isEmpty ? [] : escapedText.components(separatedBy: "|") + let actualTokens = withTokenization(of: expectedTokens.joined(), contentType: .codeCompletionSymbol) { + (mixedcaseBytes, tokenization) -> [String] in + XCTAssertLessThanOrEqual(tokenization.tokens.count, mixedcaseBytes.count) + XCTAssertEqual(tokenization.byteTokenAddresses.count, mixedcaseBytes.count) + XCTAssertEqual(tokenization.tokens.map(\.length).sum(), mixedcaseBytes.count) + + var previous: (tokenIndex: Int, indexInToken: Int)? = nil + + for cIdx in 0.. String { + try withTokenization(of: text, contentType: contentType) { mixedcaseBytes, tokenization in + return try String(bytes: mixedcaseBytes[.. Bool { + withTokenization(of: text, contentType: .codeCompletionSymbol) { mixedcaseBytes, tokenization in + tokenization.hasNonUppercaseNonDelimiterBytes + } + } + func test(text: String, expecting expectedAllUppercaseValues: Bool) { + try! XCTAssertEqual(hasNonUppercaseNonDelimiterBytes(text: text), expectedAllUppercaseValues) + } + test(text: "", expecting: false) + test(text: "A", expecting: false) + test(text: "AB", expecting: false) + test(text: "_AB", expecting: false) + test(text: "A_B", expecting: false) + test(text: "AB_", expecting: false) + test(text: "__AB", expecting: false) + test(text: "A__B", expecting: false) + test(text: "AB__", expecting: false) + + test(text: "", expecting: false) + test(text: "a", expecting: true) + test(text: "ab", expecting: true) + test(text: "_b", expecting: true) + test(text: "a_b", expecting: true) + test(text: "ab_", expecting: true) + test(text: "__ab", expecting: true) + test(text: "a__b", expecting: true) + test(text: "ab__", expecting: true) + } + + func testTokenizationIsUppercaseDetection() throws { + func allUppercaseValues(text: String) throws -> [Bool] { + withTokenization(of: text, contentType: .codeCompletionSymbol) { mixedcaseBytes, tokenization in + tokenization.tokens.map(\.allUppercase) + } + } + func test(text: String, expecting expectedAllUppercaseValues: [Bool]) { + try! XCTAssertEqual(allUppercaseValues(text: text), expectedAllUppercaseValues) + } + test(text: "", expecting: []) + test(text: "_", expecting: [false]) + test(text: "a", expecting: [false]) + test(text: "a_", expecting: [false, false]) + test(text: "ab_", expecting: [false, false]) + test(text: "abc_", expecting: [false, false]) + + test(text: "", expecting: []) + test(text: "_", expecting: [false]) + test(text: "A", expecting: [true]) + test(text: "A_", expecting: [true, false]) + test(text: "Ab_", expecting: [false, false]) + test(text: "aBc_", expecting: [false, false, false]) + test(text: "abC_", expecting: [false, true, false]) + + test(text: "__", expecting: [false, false]) + test(text: "___", expecting: [false, false, false]) + test(text: "____", expecting: [false, false, false, false]) + test(text: "NSManagedObjectContext", expecting: [true, false, false, false]) + test(text: "NSOpenGLView", expecting: [true, false, true, false]) + test(text: "NSWindow.h", expecting: [true, false, false, false]) + test(text: "translatesAutoresizingMaskIntoConstraints", expecting: [false, false, false, false, false]) + } + + func testScoring() throws { + test("NSW", precision: .thorough, prefers: "NSWindowController", over: "_n_s_w_") + test("Text", precision: .thorough, prefers: "Text", over: "NSText") + test("Text", precision: .thorough, prefers: "NSText", over: "QDTextUPP") + test("Text", precision: .thorough, prefers: "TextField", over: "VM_LIB64_SHR_TEXT") + test("Text", precision: .thorough, prefers: "TextField", over: "ABC_TEXT") + } + + func testExhaustiveScoring() throws { + func test(pattern: String, candidate: String) { + let fastScore = score(patternText: pattern, candidateText: candidate, precision: .fast) + let thoroughScore = score(patternText: pattern, candidateText: candidate, precision: .thorough) + XCTAssertGreaterThan(thoroughScore, fastScore) + } + test(pattern: "aaa", candidate: "ababaa") + test(pattern: "resetDownload", candidate: "resetDocumentDownload") + } + + func testThoroughScoringBudget() throws { + let pattern = Pattern(text: "123456789") + let decoy = "aaaaaaaAaaaAa1a2a3a4a5a6a7a8a9" + let fogOfWar = String(repeating: "_", count: 1 << 14) + Candidate.withAccessToCandidate(for: decoy + fogOfWar + "12345678a9", contentType: .codeCompletionSymbol) { + candidate in + var ranges: [Pattern.UTF8ByteRange] = [] + _ = pattern.score( + candidate: candidate.bytes, + contentType: candidate.contentType, + precision: .thorough, + captureMatchingRanges: true, + ranges: &ranges + ) + // Budget should have kept us from finding that last match. + XCTAssertEqual(ranges.count, 9) + } + } + + func test77869216() throws { + test("Im", precision: .thorough, prefers: "ImageIO", over: "IMP") + test("im", precision: .thorough, prefers: "IMP", over: "ImageIO") + test("IM", precision: .thorough, prefers: "IMP", over: "ImageIO") + } + + func testMatchingRanges() throws { + func test(_ text: String, precision: Pattern.Precision? = nil, testLowercasePattern: Bool = true) throws { + var text = text + struct Case { + var pattern: String + var candidate: String + var expectedMatchedRanges: [Range] + } + let testCase: Case = try text.withUTF8 { bytes in + var bytes = bytes[...] + var expectedMatchedRanges: [Range] = [] + var candidatePosition = 0 + var parsePosition = 0 + var candidate = Data() + var pattern = Data() + while let parseStart = try bytes.firstIndex(of: UTF8Byte("[")), + let parseEnd = try bytes.firstIndex(of: UTF8Byte("]")) + { + let candidateStart = (parseStart - parsePosition) + candidatePosition + let matchLength = (parseEnd - parseStart) - 1 + let skippedContentCharacters = parseStart - bytes.startIndex + expectedMatchedRanges.append(candidateStart ..+ matchLength) + candidate.append(contentsOf: bytes[.. UIViewController?" + ) + try test("wwwwwwwiiiiiitttttthhhhhhiiiiiinnnnnn[Within]") + try test("[t]ranslates[A]utoresizing[M]ask[I]ntoConstraints") + try test("[t]ranslates[A]utoresizing[M]ask[I]nto[C]onstraints") + try test("[f]ileurl[URL]") + try test( + "[frame]([min]Width:idealWidth:maxWidth:minHeight:idealHeight:[maxH]eight:alignment:)", + precision: .thorough + ) + /// PR Feedback Question about the relative strength of case match vs run length + try test( + "[frame]([min]Width:idealWidth:[max]Width:min[H]eight:idealHeight:maxheight:alignment:)", + precision: .thorough, + testLowercasePattern: false + ) + try test( + "[frame](minWidth:idealWidth:maxWidth:minHeight:[idealHeight]:maxheight:alignment:)", + precision: .thorough + ) + try test( + "init(ri_uuid:ri_user_time:ri_system_time:ri_pkg_idle_wkups:ri_interrupt_wkups:ri_pageins:ri_wired_size:ri_[re]sident_[size]:ri_phys_footprint:ri_proc_start_[ab]stime:ri_proc_exit_abstime:ri_child_user_time:ri_child_system_time:ri_child_pkg_idle_wkups:ri_child_interrupt_wkups:ri_child_pageins:ri_child_elapsed_abstime:)", + precision: .thorough + ) + // both .thorough and .fast should check for the exact match case, even if .thorough exhausts its budget + try test("_a_a_a_a_a_a_a_a_a_a_a_a_a_a_a_a_a_a_a_a_a_a_a_a_a_a_a_a_a_a_a_a_a_a_a_a_[aaa]") + } + + func testSearchStarts() { + func test(_ pattern: String, _ candidate: String, _ starts: [Int]) { + Candidate.withAccessToCandidate(for: candidate, contentType: .codeCompletionSymbol) { candidate in + XCTAssertEqual( + Pattern(text: pattern).test_searchStart(candidate: candidate, contentType: .codeCompletionSymbol), + starts + ) + } + } + test("", "", []) + test("a", "", []) + test("a", "a", [1]) + test("a", "aA", [1, 2]) + test("b", "aA", [2, 2]) + test("b", "bA", [2, 2]) + test("b", "aB", [1, 2]) + test("dowork", "doTheWork", [5, 5, 5, 5, 5, 9, 9, 9, 9]) + test("thework", "doTheWork", [2, 2, 5, 5, 5, 9, 9, 9, 9]) + test("dothe", "doTheWork", [2, 2, 9, 9, 9, 9, 9, 9, 9]) + } + + func testUnexpectedScoringCalls() { + func score(_ patternText: String, _ candidateText: String, precision: Pattern.Precision) -> Double { + Candidate.withAccessToCandidate(for: candidateText, contentType: .codeCompletionSymbol) { candidate in + Pattern(text: patternText).score(candidate: candidate, precision: precision) + } + } + for precision in [Pattern.Precision.fast, Pattern.Precision.thorough] { + XCTAssertEqual(score("", "", precision: precision), 1.0) + XCTAssertEqual(score("", "a", precision: precision), 1.0) + XCTAssertEqual(score("", "aa", precision: precision), 1.0) + XCTAssertEqual(score("a", "", precision: precision), 0.0) + XCTAssertEqual(score("a", "b", precision: precision), 0.0) + XCTAssertEqual(score("aa", "a", precision: precision), 0.0) + } + } + + func testExactMatchesScoreBetter() { + test("ab", precision: .thorough, prefers: "ab", over: "abc") + test("_observers", precision: .thorough, prefers: "_observers", over: "_asdfObservers") + test("_observers", precision: .thorough, prefers: "_observers", over: "_Observersasdf") + test("_observers", precision: .thorough, prefers: "_observers", over: "asdf_Observers") + test("_observers", precision: .thorough, prefers: "_observers", over: "asdf_observers") + } + + func test9225472() { + test("hasDe", precision: .thorough, prefers: "hasDetachedOccurrences", over: "bHasDesktopMgr") + test("hasDe", precision: .thorough, prefers: "hasDetachedOccurrences", over: "bHasDirectIO") + } + + func testMatchAllCapsPrefix() { + test("abcoverr", precision: .thorough, prefers: "ABCOverridingProperties", over: "indexOfViewForResizing") + test( + "abcoverr", + precision: .thorough, + prefers: "ABCOverridingProperties", + over: "setIndexOfViewForResizing:" + ) + test("abcoverr", precision: .thorough, prefers: "ABCOverridingProperties", over: "buildSettingOverrides") + } + + func test88532962() { + let propertyResult = SemanticScoredText( + "textColor", + SemanticClassification( + availability: .available, + completionKind: .variable, + flair: [], + moduleProximity: .imported(distance: 1), + popularity: .none, + scopeProximity: .container, + structuralProximity: .inapplicable, + synchronicityCompatibility: .inapplicable, + typeCompatibility: .compatible + ).score + ) + let typeResult = SemanticScoredText( + "Text", + SemanticClassification( + availability: .available, + completionKind: .type, + flair: [], + moduleProximity: .imported(distance: 1), + popularity: .none, + scopeProximity: .global, + structuralProximity: .inapplicable, + synchronicityCompatibility: .inapplicable, + typeCompatibility: .unrelated + ).score + ) + test(completion: propertyResult, alwaysBeats: [typeResult]) + } + + func test88530047() { + let propertyResult = SemanticScoredText( + "dataSource", + SemanticClassification( + availability: .available, + completionKind: .variable, + flair: [], + moduleProximity: .imported(distance: 0), + popularity: .none, + scopeProximity: .container, + structuralProximity: .inapplicable, + synchronicityCompatibility: .inapplicable, + typeCompatibility: .inapplicable + ).score + ) + let typeResult = SemanticScoredText( + "Data", + SemanticClassification( + availability: .available, + completionKind: .type, + flair: [], + moduleProximity: .imported(distance: 0), + popularity: .none, + scopeProximity: .global, + structuralProximity: .inapplicable, + synchronicityCompatibility: .inapplicable, + typeCompatibility: .inapplicable + ).score + ) + test(completion: propertyResult, alwaysBeats: [typeResult]) + } + + func testPreferConsecutiveMatchesBetweenPatternAndCandidate() { + test( + "ABStorDocument", + precision: .thorough, + prefers: "ABStoryboardDocument", + over: "ABPrintablePListForDocumentClasses" + ) + } + + func test(completion expectedWinner: SemanticScoredText, alwaysBeats decoys: [SemanticScoredText]) { + expectedWinner.text.enumeratePrefixes(includeLowercased: true) { partialPattern in + for decoy in decoys { + test(partialPattern, precision: .thorough, prefers: expectedWinner, over: decoy) + } + } + } + + func test68718765() { + let localVariable = SemanticScoredText( + "localVariable", + .partial( + availability: .available, + completionKind: .variable, + flair: [], + moduleProximity: .imported(distance: 0), + popularity: .none, + scopeProximity: .local, + structuralProximity: .sdk, + synchronicityCompatibility: .compatible, + typeCompatibility: .inapplicable + ) + ) + let globalEnum = SemanticScoredText( + "errSecCSBadLVArch", + .partial( + availability: .available, + completionKind: .enumCase, + flair: [], + moduleProximity: .imported(distance: 3), + popularity: .none, + scopeProximity: .global, + structuralProximity: .sdk, + synchronicityCompatibility: .compatible, + typeCompatibility: .inapplicable + ) + ) + test(completion: localVariable, alwaysBeats: [globalEnum]) + } + + func test88694877() { + let classResult = SemanticScoredText( + "NSLayoutConstraint", + SemanticClassification( + availability: .available, + completionKind: .type, + flair: [], + moduleProximity: .imported(distance: 0), + popularity: .none, + scopeProximity: .global, + structuralProximity: .inapplicable, + synchronicityCompatibility: .inapplicable, + typeCompatibility: .inapplicable + ).score + ) + let headerGuardResult = SemanticScoredText( + "NSLAYOUTCONSTRAINT_H", + SemanticClassification( + availability: .available, + completionKind: .variable, + flair: [], + moduleProximity: .imported(distance: 0), + popularity: .none, + scopeProximity: .global, + structuralProximity: .inapplicable, + synchronicityCompatibility: .inapplicable, + typeCompatibility: .inapplicable + ).score + ) + test(completion: classResult, alwaysBeats: [headerGuardResult]) + } + + func test88694877_2() { + test("fileName", precision: .thorough, prefers: "fileName", over: "filename") + } + + func test77934625_1() { + let popularity = PopularityIndex(referencePercentages: [ + "SwiftUI": [ + "View": 0.250, + "VStack": 0.025, + "HStack": 0.025, + "Button": 0.025, + "Text": 0.025, + "Image": 0.025, + "Label": 0.025, + ] + ]) + let view = SemanticScoredText( + "View", + SemanticClassification( + availability: .available, + completionKind: .type, + flair: [.rareTypeAtCurrentPosition], + moduleProximity: .imported(distance: 1), + popularity: popularity.popularity(of: "SwiftUI.View"), + scopeProximity: .global, + structuralProximity: .sdk, + synchronicityCompatibility: .compatible, + typeCompatibility: .inapplicable + ).score + ) + let vstack = SemanticScoredText( + "VStack", + SemanticClassification( + availability: .available, + completionKind: .type, + flair: [], + moduleProximity: .imported(distance: 1), + popularity: popularity.popularity(of: "SwiftUI.VStack"), + scopeProximity: .global, + structuralProximity: .sdk, + synchronicityCompatibility: .compatible, + typeCompatibility: .inapplicable + ).score + ) + test("V", precision: .thorough, prefers: vstack, over: view) + } + + func test77934625_2() { + let popularity = PopularityIndex(referencePercentages: [ + "Swift": [ + "Array": 0.250, + "Int": 0.125, + "Dictionary": 0.125, + "Bool": 0.125, + "Set": 0.125, + "Collection": 0.125, + "Sequence": 0.120, + "Encoder": 0.005, + ] + ]) + let enumResult = SemanticScoredText( + "enum", + SemanticClassification( + availability: .available, + completionKind: .keyword, + flair: [.commonKeywordAtCurrentPosition], + moduleProximity: .inapplicable, + popularity: .none, + scopeProximity: .inapplicable, + structuralProximity: .inapplicable, + synchronicityCompatibility: .inapplicable, + typeCompatibility: .inapplicable + ).score + ) + let extensionResult = SemanticScoredText( + "extension", + SemanticClassification( + availability: .available, + completionKind: .keyword, + flair: [.commonKeywordAtCurrentPosition], + moduleProximity: .inapplicable, + popularity: .none, + scopeProximity: .inapplicable, + structuralProximity: .inapplicable, + synchronicityCompatibility: .inapplicable, + typeCompatibility: .inapplicable + ).score + ) + let encoderResult = SemanticScoredText( + "Encoder", + SemanticClassification( + availability: .available, + completionKind: .type, + flair: [.expressionAtNonScriptOrMainFileScope], + moduleProximity: .imported(distance: 1), + popularity: popularity.popularity(of: "Swift.Encoder"), + scopeProximity: .global, + structuralProximity: .sdk, + synchronicityCompatibility: .compatible, + typeCompatibility: .inapplicable + ).score + ) + test("e", precision: .thorough, prefers: enumResult, over: encoderResult) + test("e", precision: .thorough, prefers: extensionResult, over: encoderResult) + } + + func testPreferLessTokens() { + let distantUnrelatedGlobal = SemanticClassification( + availability: .available, + completionKind: .variable, + flair: [], + moduleProximity: .imported(distance: 3), + popularity: .none, + scopeProximity: .global, + structuralProximity: .sdk, + synchronicityCompatibility: .compatible, + typeCompatibility: .unrelated + ).score + let distantType = SemanticClassification( + availability: .available, + completionKind: .type, + flair: [], + moduleProximity: .imported(distance: 3), + popularity: .none, + scopeProximity: .global, + structuralProximity: .sdk, + synchronicityCompatibility: .compatible, + typeCompatibility: .inapplicable + ).score + let nearType = SemanticClassification( + availability: .available, + completionKind: .type, + flair: [], + moduleProximity: .imported(distance: 1), + popularity: .none, + scopeProximity: .global, + structuralProximity: .sdk, + synchronicityCompatibility: .compatible, + typeCompatibility: .inapplicable + ).score + let nearInheritedInstanceMethod = SemanticClassification( + availability: .available, + completionKind: .function, + flair: [], + moduleProximity: .imported(distance: 1), + popularity: .none, + scopeProximity: .inheritedContainer, + structuralProximity: .sdk, + synchronicityCompatibility: .compatible, + typeCompatibility: .inapplicable + ).score + let nearInheritedUnrelatedInstanceMethod = SemanticClassification( + availability: .available, + completionKind: .function, + flair: [], + moduleProximity: .imported(distance: 1), + popularity: .none, + scopeProximity: .inheritedContainer, + structuralProximity: .sdk, + synchronicityCompatibility: .compatible, + typeCompatibility: .unrelated + ).score + let keyword = SemanticClassification( + availability: .available, + completionKind: .keyword, + flair: [], + moduleProximity: .inapplicable, + popularity: .none, + scopeProximity: .inapplicable, + structuralProximity: .inapplicable, + synchronicityCompatibility: .inapplicable, + typeCompatibility: .inapplicable + ).score + let decoys = [ + SemanticScoredText("T_FMT", distantUnrelatedGlobal), + SemanticScoredText("T_FMT_AMPM", distantUnrelatedGlobal), + SemanticScoredText("TScriptingSizeResource", distantType), + SemanticScoredText("TAB3", distantUnrelatedGlobal), + SemanticScoredText("TAB2", distantUnrelatedGlobal), + SemanticScoredText("TAB1", distantUnrelatedGlobal), + SemanticScoredText("TAB0", distantUnrelatedGlobal), + SemanticScoredText("TH_URG", distantUnrelatedGlobal), + SemanticScoredText("TH_SYN", distantUnrelatedGlobal), + SemanticScoredText("TH_RST", distantUnrelatedGlobal), + SemanticScoredText("TH_FIN", distantUnrelatedGlobal), + SemanticScoredText("TH_ECE", distantUnrelatedGlobal), + SemanticScoredText("TH_CWR", distantUnrelatedGlobal), + SemanticScoredText("TH_ACK", distantUnrelatedGlobal), + SemanticScoredText("TS_LNCH", distantUnrelatedGlobal), + SemanticScoredText("TS_BUSY", distantUnrelatedGlobal), + SemanticScoredText("TS_BKSL", distantUnrelatedGlobal), + SemanticScoredText("TR_ZFOD", distantUnrelatedGlobal), + SemanticScoredText("TR_MALL", distantUnrelatedGlobal), + SemanticScoredText("TextField", nearType), + SemanticScoredText("TextEditor", nearType), + SemanticScoredText("TextAlignment", nearType), + SemanticScoredText("TextFieldStyle", nearType), + SemanticScoredText("TextOutputStream", nearType), + SemanticScoredText("TextEditingCommands", nearType), + SemanticScoredText("TextRange", distantType), + SemanticScoredText("TextFormattingCommands", nearType), + SemanticScoredText("TextOutputStreamable", nearType), + SemanticScoredText("TextRangePtr", distantType), + SemanticScoredText("multilineTextAlignment(:)", nearInheritedInstanceMethod), + SemanticScoredText("TextRangeArray", distantType), + SemanticScoredText("TextRangeHandle", distantType), + SemanticScoredText("accessibilityTextContentType(:)", nearInheritedUnrelatedInstanceMethod), + SemanticScoredText("TextRangeArrayPtr", distantType), + SemanticScoredText("TextBreakLocatorRef", distantType), + SemanticScoredText("TextRangeArrayHandle", distantType), + SemanticScoredText("TernaryPrecedence", keyword), + ] + let text = SemanticScoredText("Text", nearType) + test(completion: text, alwaysBeats: decoys) + } + + func testPenalizeGlobalVariables() { + let distantEnumCase = SemanticClassification( + availability: .available, + completionKind: .enumCase, + flair: [], + moduleProximity: .imported(distance: 3), + popularity: .none, + scopeProximity: .global, + structuralProximity: .sdk, + synchronicityCompatibility: .compatible, + typeCompatibility: .inapplicable + ).score + let distantGlobal = SemanticClassification( + availability: .available, + completionKind: .variable, + flair: [], + moduleProximity: .imported(distance: 3), + popularity: .none, + scopeProximity: .global, + structuralProximity: .sdk, + synchronicityCompatibility: .compatible, + typeCompatibility: .inapplicable + ).score + let distantType = SemanticClassification( + availability: .available, + completionKind: .type, + flair: [], + moduleProximity: .imported(distance: 3), + popularity: .none, + scopeProximity: .global, + structuralProximity: .sdk, + synchronicityCompatibility: .compatible, + typeCompatibility: .inapplicable + ).score + let nearType = SemanticClassification( + availability: .available, + completionKind: .type, + flair: [], + moduleProximity: .imported(distance: 1), + popularity: .none, + scopeProximity: .global, + structuralProximity: .sdk, + synchronicityCompatibility: .compatible, + typeCompatibility: .inapplicable + ).score + let decoys = [ + SemanticScoredText("exUserBreak", distantEnumCase), + SemanticScoredText("EX_OK", distantGlobal), + SemanticScoredText("EX__MAX", distantGlobal), + SemanticScoredText("EX_IOERR", distantGlobal), + SemanticScoredText("eHD", distantEnumCase), + SemanticScoredText("eIP", distantEnumCase), + SemanticScoredText("eADB", distantEnumCase), + SemanticScoredText("eBus", distantEnumCase), + SemanticScoredText("eDVD", distantEnumCase), + SemanticScoredText("eLCD", distantEnumCase), + SemanticScoredText("ePPP", distantEnumCase), + SemanticScoredText("eUSB", distantEnumCase), + SemanticScoredText("eIrDA", distantEnumCase), + SemanticScoredText("eSCSI", distantEnumCase), + SemanticScoredText("extFSErr", distantEnumCase), + SemanticScoredText("EXTA", distantGlobal), + SemanticScoredText("extractErr", distantEnumCase), + SemanticScoredText("extendedBlock", distantEnumCase), + SemanticScoredText("extendedBlock", distantEnumCase), + SemanticScoredText("extendedBlockLen", distantEnumCase), + SemanticScoredText("extern_proc", distantType), + SemanticScoredText("ExtendedGraphemeClusterType", nearType), + SemanticScoredText("extentrecord", distantType), + SemanticScoredText("extension_data_format", distantType), + ] + let keyword = SemanticClassification( + availability: .available, + completionKind: .keyword, + flair: [], + moduleProximity: .inapplicable, + popularity: .none, + scopeProximity: .inapplicable, + structuralProximity: .inapplicable, + synchronicityCompatibility: .inapplicable, + typeCompatibility: .inapplicable + ).score + let extensionKeyword = SemanticScoredText("extension", keyword) + test(completion: extensionKeyword, alwaysBeats: decoys) + } + + func test74888915() { + let string = SemanticScoredText( + "String", + .partial( + availability: .available, + completionKind: .type, + flair: [], + moduleProximity: .imported(distance: 1), + popularity: .init(scoreComponent: 1.1), + scopeProximity: .global, + structuralProximity: .sdk, + synchronicityCompatibility: .inapplicable, + typeCompatibility: .inapplicable + ) + ) + let decoys = [ + SemanticScoredText( + "S_OK", + .partial( + availability: .available, + completionKind: .variable, + flair: [], + moduleProximity: .imported(distance: 1), + popularity: .none, + scopeProximity: .global, + structuralProximity: .sdk, + synchronicityCompatibility: .compatible, + typeCompatibility: .inapplicable + ) + ), + SemanticScoredText( + "s_zerofill", + .partial( + availability: .available, + completionKind: .variable, + flair: [], + moduleProximity: .imported(distance: 3), + popularity: .none, + scopeProximity: .global, + structuralProximity: .sdk, + synchronicityCompatibility: .compatible, + typeCompatibility: .inapplicable + ) + ), + SemanticScoredText( + "ST_NOSUID", + .partial( + availability: .available, + completionKind: .variable, + flair: [], + moduleProximity: .imported(distance: 3), + popularity: .none, + scopeProximity: .global, + structuralProximity: .sdk, + synchronicityCompatibility: .compatible, + typeCompatibility: .inapplicable + ) + ), + SemanticScoredText( + "ST_RDONLY", + .partial( + availability: .available, + completionKind: .variable, + flair: [], + moduleProximity: .imported(distance: 3), + popularity: .none, + scopeProximity: .global, + structuralProximity: .sdk, + synchronicityCompatibility: .compatible, + typeCompatibility: .inapplicable + ) + ), + SemanticScoredText( + "strUserBreak", + .partial( + availability: .available, + completionKind: .enumCase, + flair: [], + moduleProximity: .imported(distance: 3), + popularity: .none, + scopeProximity: .global, + structuralProximity: .sdk, + synchronicityCompatibility: .compatible, + typeCompatibility: .inapplicable + ) + ), + SemanticScoredText( + "strlen()", + .partial( + availability: .available, + completionKind: .function, + flair: [], + moduleProximity: .imported(distance: 2), + popularity: .none, + scopeProximity: .global, + structuralProximity: .sdk, + synchronicityCompatibility: .compatible, + typeCompatibility: .inapplicable + ) + ), + SemanticScoredText( + "STClass", + .partial( + availability: .available, + completionKind: .variable, + flair: [], + moduleProximity: .imported(distance: 3), + popularity: .none, + scopeProximity: .global, + structuralProximity: .sdk, + synchronicityCompatibility: .compatible, + typeCompatibility: .inapplicable + ) + ), + SemanticScoredText( + "STHeader", + .partial( + availability: .available, + completionKind: .type, + flair: [], + moduleProximity: .imported(distance: 3), + popularity: .none, + scopeProximity: .global, + structuralProximity: .sdk, + synchronicityCompatibility: .inapplicable, + typeCompatibility: .inapplicable + ) + ), + SemanticScoredText( + "StrideTo", + .partial( + availability: .available, + completionKind: .type, + flair: [], + moduleProximity: .imported(distance: 1), + popularity: .none, + scopeProximity: .global, + structuralProximity: .sdk, + synchronicityCompatibility: .inapplicable, + typeCompatibility: .inapplicable + ) + ), + SemanticScoredText( + "StrideThrough", + .partial( + availability: .available, + completionKind: .type, + flair: [], + moduleProximity: .imported(distance: 1), + popularity: .none, + scopeProximity: .global, + structuralProximity: .sdk, + synchronicityCompatibility: .inapplicable, + typeCompatibility: .inapplicable + ) + ), + ] + test(completion: string, alwaysBeats: decoys) + } + + func test90527878() { + let swiftUIText = SemanticScoredText( + "Text", + .partial( + availability: .available, + completionKind: .type, + flair: [], + moduleProximity: .imported(distance: 1), + popularity: .none, + scopeProximity: .global, + structuralProximity: .sdk, + synchronicityCompatibility: .inapplicable, + typeCompatibility: .compatible + ) + ) + let decoys = [ + SemanticScoredText( + "task(id:_)", + .partial( + availability: .available, + completionKind: .function, + flair: [.swiftUIModifierOnSelfWhileBuildingSelf], + moduleProximity: .imported(distance: 1), + popularity: .none, + scopeProximity: .container, + structuralProximity: .sdk, + synchronicityCompatibility: .compatible, + typeCompatibility: .compatible + ) + ), + SemanticScoredText( + "tint(_:)", + .partial( + availability: .available, + completionKind: .function, + flair: [.swiftUIModifierOnSelfWhileBuildingSelf], + moduleProximity: .imported(distance: 1), + popularity: .none, + scopeProximity: .container, + structuralProximity: .sdk, + synchronicityCompatibility: .compatible, + typeCompatibility: .compatible + ) + ), + SemanticScoredText( + "tabItem(_:)", + .partial( + availability: .available, + completionKind: .function, + flair: [.swiftUIModifierOnSelfWhileBuildingSelf], + moduleProximity: .imported(distance: 1), + popularity: .none, + scopeProximity: .container, + structuralProximity: .sdk, + synchronicityCompatibility: .compatible, + typeCompatibility: .compatible + ) + ), + SemanticScoredText( + "tracking(_:)", + .partial( + availability: .available, + completionKind: .function, + flair: [.swiftUIModifierOnSelfWhileBuildingSelf], + moduleProximity: .imported(distance: 1), + popularity: .none, + scopeProximity: .container, + structuralProximity: .sdk, + synchronicityCompatibility: .compatible, + typeCompatibility: .compatible + ) + ), + SemanticScoredText( + "textCase(_:)", + .partial( + availability: .available, + completionKind: .function, + flair: [.swiftUIModifierOnSelfWhileBuildingSelf], + moduleProximity: .imported(distance: 1), + popularity: .none, + scopeProximity: .container, + structuralProximity: .sdk, + synchronicityCompatibility: .compatible, + typeCompatibility: .compatible + ) + ), + SemanticScoredText( + "textSelection(_:)", + .partial( + availability: .available, + completionKind: .function, + flair: [.swiftUIModifierOnSelfWhileBuildingSelf], + moduleProximity: .imported(distance: 1), + popularity: .none, + scopeProximity: .container, + structuralProximity: .sdk, + synchronicityCompatibility: .compatible, + typeCompatibility: .compatible + ) + ), + SemanticScoredText( + "text()", + .partial( + availability: .available, + completionKind: .function, + flair: [.swiftUIModifierOnSelfWhileBuildingSelf], + moduleProximity: .imported(distance: 1), + popularity: .none, + scopeProximity: .container, + structuralProximity: .sdk, + synchronicityCompatibility: .compatible, + typeCompatibility: .compatible + ) + ), + ] + test(completion: swiftUIText, alwaysBeats: decoys) + } + + func testTextScoreOrdering() { + typealias TextScore = Pattern.TextScore + XCTAssertLessThan(TextScore(value: 0, falseStarts: 0), TextScore(value: 1, falseStarts: 0)) + XCTAssertLessThan(TextScore(value: 0, falseStarts: 1), TextScore(value: 0, falseStarts: 0)) + XCTAssertLessThan(TextScore(value: 1, falseStarts: 1), TextScore(value: 1, falseStarts: 0)) + } + + func testFalseStartCounts() { + func falseStarts( + pattern patternText: String, + candidate: String, + contentType: CandidateBatch.ContentType = .codeCompletionSymbol + ) -> Int { + candidate.withUncachedUTF8Bytes { candidateUTF8Bytes in + UnsafeStackAllocator.withUnsafeStackAllocator { allocator in + Pattern(text: patternText).score( + candidate: candidateUTF8Bytes, + contentType: contentType, + precision: .thorough, + allocator: &allocator + ).falseStarts + } + } + } + + XCTAssertEqual(falseStarts(pattern: "per", candidate: "person"), 0) + XCTAssertEqual(falseStarts(pattern: "per", candidate: "super"), 1) + XCTAssertEqual(falseStarts(pattern: "per", candidate: "pear"), 2) + XCTAssertEqual(falseStarts(pattern: "tamic", candidate: "translatesAutoresizingMaskIntoConstraints"), 0) + XCTAssertEqual(falseStarts(pattern: "amic", candidate: "translatesAutoresizingMaskIntoConstraints"), 3) + XCTAssertEqual(falseStarts(pattern: "args", candidate: "NS(all:red:go:slow:)"), 3) + XCTAssertEqual(falseStarts(pattern: "segmentControl", candidate: "segmentedControl"), 1) + XCTAssertEqual(falseStarts(pattern: "hsta", candidate: "ABC_FHSTAT"), 1) + XCTAssertEqual(falseStarts(pattern: "HSta", candidate: "HStack"), 0) + XCTAssertEqual(falseStarts(pattern: "HSta", candidate: "LazyHStack"), 0) + XCTAssertEqual(falseStarts(pattern: "resetDownload", candidate: "resetDocumentDownload"), 0) + XCTAssertEqual(falseStarts(pattern: "resetDown", candidate: "resetDocumentDownload"), 0) + XCTAssertEqual(falseStarts(pattern: "moc", candidate: "NSManagedObjectContext"), 0) + XCTAssertEqual(falseStarts(pattern: "nmoc", candidate: "NSManagedObjectContext"), 0) + XCTAssertEqual(falseStarts(pattern: "nsmoc", candidate: "NSManagedObjectContext"), 0) + + // rdar://87776071 (Typing `reduce` suggested 5,000 character struct initializer) + let tcpstatInitializer = + "tcpstat(tcps_connattempt:tcps_accepts:tcps_connects:tcps_drops:tcps_conndrops:tcps_closed:tcps_segstimed:tcps_rttupdated:tcps_delack:tcps_timeoutdrop:tcps_rexmttimeo:tcps_persisttimeo:tcps_keeptimeo:tcps_keepprobe:tcps_keepdrops:tcps_sndtotal:tcps_sndpack:tcps_sndbyte:tcps_sndrexmitpack:tcps_sndrexmitbyte:tcps_sndacks:tcps_sndprobe:tcps_sndurg:tcps_sndwinup:tcps_sndctrl:tcps_rcvtotal:tcps_rcvpack:tcps_rcvbyte:tcps_rcvbadsum:tcps_rcvbadoff:tcps_rcvmemdrop:tcps_rcvshort:tcps_rcvduppack:tcps_rcvdupbyte:tcps_rcvpartduppack:tcps_rcvpartdupbyte:tcps_rcvoopack:tcps_rcvoobyte:tcps_rcvpackafterwin:tcps_rcvbyteafterwin:tcps_rcvafterclose:tcps_rcvwinprobe:tcps_rcvdupack:tcps_rcvacktoomuch:tcps_rcvackpack:tcps_rcvackbyte:tcps_rcvwinupd:tcps_pawsdrop:tcps_predack:tcps_preddat:tcps_pcbcachemiss:tcps_cachedrtt:tcps_cachedrttvar:tcps_cachedssthresh:tcps_usedrtt:tcps_usedrttvar:tcps_usedssthresh:tcps_persistdrop:tcps_badsyn:tcps_mturesent:tcps_listendrop:tcps_synchallenge:tcps_rstchallenge:tcps_minmssdrops:tcps_sndrexmitbad:tcps_badrst:tcps_sc_added:tcps_sc_retransmitted:tcps_sc_dupsyn:tcps_sc_dropped:tcps_sc_completed:tcps_sc_bucketoverflow:tcps_sc_cacheoverflow:tcps_sc_reset:tcps_sc_stale:tcps_sc_aborted:tcps_sc_badack:tcps_sc_unreach:tcps_sc_zonefail:tcps_sc_sendcookie:tcps_sc_recvcookie:tcps_hc_added:tcps_hc_bucketoverflow:tcps_sack_recovery_episode:tcps_sack_rexmits:tcps_sack_rexmit_bytes:tcps_sack_rcv_blocks:tcps_sack_send_blocks:tcps_sack_sboverflow:tcps_bg_rcvtotal:tcps_rxtfindrop:tcps_fcholdpacket:tcps_limited_txt:tcps_early_rexmt:tcps_sack_ackadv:tcps_rcv_swcsum:tcps_rcv_swcsum_bytes:tcps_rcv6_swcsum:tcps_rcv6_swcsum_bytes:tcps_snd_swcsum:tcps_snd_swcsum_bytes:tcps_snd6_swcsum:tcps_snd6_swcsum_bytes:tcps_unused_1:tcps_unused_2:tcps_unused_3:tcps_invalid_mpcap:tcps_invalid_joins:tcps_mpcap_fallback:tcps_join_fallback:tcps_estab_fallback:tcps_invalid_opt:tcps_mp_outofwin:tcps_mp_reducedwin:tcps_mp_badcsum:tcps_mp_oodata:tcps_mp_switches:tcps_mp_rcvtotal:tcps_mp_rcvbytes:tcps_mp_sndpacks:tcps_mp_sndbytes:tcps_join_rxmts:tcps_tailloss_rto:tcps_reordered_pkts:tcps_recovered_pkts:tcps_pto:tcps_rto_after_pto:tcps_tlp_recovery:tcps_tlp_recoverlastpkt:tcps_ecn_client_success:tcps_ecn_recv_ece:tcps_ecn_sent_ece:tcps_detect_reordering:tcps_delay_recovery:tcps_avoid_rxmt:tcps_unnecessary_rxmt:tcps_nostretchack:tcps_rescue_rxmt:tcps_pto_in_recovery:tcps_pmtudbh_reverted:tcps_dsack_disable:tcps_dsack_ackloss:tcps_dsack_badrexmt:tcps_dsack_sent:tcps_dsack_recvd:tcps_dsack_recvd_old:tcps_mp_sel_symtomsd:tcps_mp_sel_rtt:tcps_mp_sel_rto:tcps_mp_sel_peer:tcps_mp_num_probes:tcps_mp_verdowngrade:tcps_drop_after_sleep:tcps_probe_if:tcps_probe_if_conflict:tcps_ecn_client_setup:tcps_ecn_server_setup:tcps_ecn_server_success:tcps_ecn_ace_syn_not_ect:tcps_ecn_ace_syn_ect1:tcps_ecn_ace_syn_ect0:tcps_ecn_ace_syn_ce:tcps_ecn_lost_synack:tcps_ecn_lost_syn:tcps_ecn_not_supported:tcps_ecn_recv_ce:tcps_ecn_ace_recv_ce:tcps_ecn_conn_recv_ce:tcps_ecn_conn_recv_ece:tcps_ecn_conn_plnoce:tcps_ecn_conn_pl_ce:tcps_ecn_conn_nopl_ce:tcps_ecn_fallback_synloss:tcps_ecn_fallback_reorder:tcps_ecn_fallback_ce:tcps_tfo_syn_data_rcv:tcps_tfo_cookie_req_rcv:tcps_tfo_cookie_sent:tcps_tfo_cookie_invalid:tcps_tfo_cookie_req:tcps_tfo_cookie_rcv:tcps_tfo_syn_data_sent:tcps_tfo_syn_data_acked:tcps_tfo_syn_loss:tcps_tfo_blackhole:tcps_tfo_cookie_wrong:tcps_tfo_no_cookie_rcv:tcps_tfo_heuristics_disable:tcps_tfo_sndblackhole:tcps_mss_to_default:tcps_mss_to_medium:tcps_mss_to_low:tcps_ecn_fallback_droprst:tcps_ecn_fallback_droprxmt:tcps_ecn_fallback_synrst:tcps_mptcp_rcvmemdrop:tcps_mptcp_rcvduppack:tcps_mptcp_rcvpackafterwin:tcps_timer_drift_le_1_ms:tcps_timer_drift_le_10_ms:tcps_timer_drift_le_20_ms:tcps_timer_drift_le_50_ms:tcps_timer_drift_le_100_ms:tcps_timer_drift_le_200_ms:tcps_timer_drift_le_500_ms:tcps_timer_drift_le_1000_ms:tcps_timer_drift_gt_1000_ms:tcps_mptcp_handover_attempt:tcps_mptcp_interactive_attempt:tcps_mptcp_aggregate_attempt:tcps_mptcp_fp_handover_attempt:tcps_mptcp_fp_interactive_attempt:tcps_mptcp_fp_aggregate_attempt:tcps_mptcp_heuristic_fallback:tcps_mptcp_fp_heuristic_fallback:tcps_mptcp_handover_success_wifi:tcps_mptcp_handover_success_cell:tcps_mptcp_interactive_success:tcps_mptcp_aggregate_success:tcps_mptcp_fp_handover_success_wifi:tcps_mptcp_fp_handover_success_cell:tcps_mptcp_fp_interactive_success:tcps_mptcp_fp_aggregate_success:tcps_mptcp_handover_cell_from_wifi:tcps_mptcp_handover_wifi_from_cell:tcps_mptcp_interactive_cell_from_wifi:tcps_mptcp_handover_cell_bytes:tcps_mptcp_interactive_cell_bytes:tcps_mptcp_aggregate_cell_bytes:tcps_mptcp_handover_all_bytes:tcps_mptcp_interactive_all_bytes:tcps_mptcp_aggregate_all_bytes:tcps_mptcp_back_to_wifi:tcps_mptcp_wifi_proxy:tcps_mptcp_cell_proxy:tcps_ka_offload_drops:tcps_mptcp_triggered_cell:tcps_fin_timeout_drops:)" + XCTAssertEqual(falseStarts(pattern: "reduce", candidate: tcpstatInitializer), 1) + XCTAssertEqual(falseStarts(pattern: "tcpst", candidate: tcpstatInitializer), 0) + XCTAssertEqual(falseStarts(pattern: "willDis", candidate: "tableView(:willDisplayCell:row:)"), 0) + XCTAssertEqual( + falseStarts(pattern: "toolTip", candidate: "outlineView?(toolTipFor:rect:tableColumn:item:mouseLocation:)"), + 0 + ) + XCTAssertEqual( + falseStarts( + pattern: "mouseLocation", + candidate: "outlineView?(toolTipFor:rect:tableColumn:item:mouseLocation:)" + ), + 0 + ) + + XCTAssertEqual(falseStarts(pattern: "gvh", candidate: "SATGraphView.h", contentType: .fileName), 0) + XCTAssertEqual(falseStarts(pattern: "sgv", candidate: "SATGraphView.h", contentType: .fileName), 0) + XCTAssertEqual(falseStarts(pattern: "sgv", candidate: "SATGraphView.h", contentType: .fileName), 0) + XCTAssertEqual( + falseStarts(pattern: "skgv", candidate: "SATGraphView.h", contentType: .fileName), + 0 + ) + XCTAssertEqual( + falseStarts(pattern: "sktgv", candidate: "SATGraphView.h", contentType: .fileName), + 0 + ) + XCTAssertEqual( + falseStarts(pattern: "sgvh", candidate: "SATGraphView.h", contentType: .fileName), + 0 + ) + XCTAssertEqual(falseStarts(pattern: "gv", candidate: "SATGraphView.h", contentType: .fileName), 1) + + XCTAssertEqual( + falseStarts(pattern: "idiodic", candidate: "IDAHO_IOMATIC_DECOY_ICE", contentType: .codeCompletionSymbol), + 3 + ) + + // Good Matches + + for contentType in [ + ContentType.projectSymbol, ContentType.codeCompletionSymbol, ContentType.fileName, + ] { + let candidate = (contentType == .fileName) ? "NSOpenGLView.h" : "NSOpenGLView" + XCTAssertEqual(falseStarts(pattern: "nsoglv", candidate: candidate, contentType: contentType), 0) + XCTAssertEqual(falseStarts(pattern: "nsogv", candidate: candidate, contentType: contentType), 0) + XCTAssertEqual(falseStarts(pattern: "noglv", candidate: candidate, contentType: contentType), 0) + XCTAssertEqual(falseStarts(pattern: "oglv", candidate: candidate, contentType: contentType), 0) + XCTAssertEqual(falseStarts(pattern: "glv", candidate: candidate, contentType: contentType), 0) + XCTAssertEqual(falseStarts(pattern: "noglv", candidate: candidate, contentType: contentType), 0) + // False starts + XCTAssertEqual(falseStarts(pattern: "soglv", candidate: candidate, contentType: contentType), 2) + XCTAssertEqual(falseStarts(pattern: "olv", candidate: candidate, contentType: contentType), 2) + XCTAssertEqual(falseStarts(pattern: "nsov", candidate: candidate, contentType: contentType), 1) + + } + } +} diff --git a/Tests/CompletionScoringTests/RejectionFilterTests.swift b/Tests/CompletionScoringTests/RejectionFilterTests.swift new file mode 100644 index 000000000..e585677d8 --- /dev/null +++ b/Tests/CompletionScoringTests/RejectionFilterTests.swift @@ -0,0 +1,118 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import CompletionScoring +import XCTest + +class RejectionFilterTests: XCTestCase { + func testMatches() throws { + func test(pattern: RejectionFilter, candidate: RejectionFilter, match: RejectionFilter.Match) { + XCTAssertEqual(RejectionFilter.match(pattern: pattern, candidate: candidate), match) + } + + func test(pattern: String, candidate: String, match: RejectionFilter.Match) { + test(pattern: RejectionFilter(string: pattern), candidate: RejectionFilter(string: candidate), match: match) + } + + func test(pattern: [UInt8], candidate: [UInt8], match: RejectionFilter.Match) { + test(pattern: RejectionFilter(bytes: pattern), candidate: RejectionFilter(bytes: candidate), match: match) + } + + test(pattern: "", candidate: "", match: .maybe) + test(pattern: "", candidate: "a", match: .maybe) + test(pattern: "a", candidate: "a", match: .maybe) + test(pattern: "a", candidate: "aa", match: .maybe) + test(pattern: "aa", candidate: "a", match: .maybe) + test(pattern: "b", candidate: "a", match: .no) + test(pattern: "b", candidate: "ba", match: .maybe) + test(pattern: "b", candidate: "ab", match: .maybe) + test(pattern: "ba", candidate: "a", match: .no) + test(pattern: "$", candidate: "$", match: .maybe) + test(pattern: "<", candidate: "<", match: .maybe) + test(pattern: "a", candidate: "Z", match: .no) + test(pattern: "z", candidate: "Z", match: .maybe) + test(pattern: "_", candidate: "a", match: .no) + + let allBytes = UInt8.min...UInt8.max + for byte in allBytes { + test(pattern: [], candidate: [byte], match: .maybe) + test(pattern: [byte], candidate: [], match: .no) + test(pattern: [byte], candidate: [byte], match: .maybe) + test(pattern: [byte], candidate: [byte, byte], match: .maybe) + test(pattern: [byte, byte], candidate: [byte], match: .maybe) + test(pattern: [byte, byte], candidate: [byte, byte], match: .maybe) + } + + for letter in UTF8Byte.lowercaseAZ { + test(pattern: [letter], candidate: [letter], match: .maybe) + test(pattern: [letter.uppercasedUTF8Byte], candidate: [letter], match: .maybe) + test(pattern: [letter], candidate: [letter.uppercasedUTF8Byte], match: .maybe) + test(pattern: [letter.uppercasedUTF8Byte], candidate: [letter.uppercasedUTF8Byte], match: .maybe) + test(pattern: [UTF8Byte.cUnderscore], candidate: [letter.uppercasedUTF8Byte], match: .no) + test(pattern: [letter.uppercasedUTF8Byte], candidate: [UTF8Byte.cUnderscore], match: .no) + } + + for b1 in allBytes { + for b2 in allBytes { + test(pattern: [b1], candidate: [b1, b2], match: .maybe) + test(pattern: [b1], candidate: [b2, b1], match: .maybe) + } + } + } + + func testMatchesExhaustively() { + @Sendable func matches(pattern: UnsafeBufferPointer, candidate: UnsafeBufferPointer) -> Bool { + var pIdx = 0 + var cIdx = 0 + while (cIdx != candidate.count) && (pIdx != pattern.count) { + if pattern[pIdx].lowercasedUTF8Byte == candidate[cIdx].lowercasedUTF8Byte { + pIdx += 1 + } + cIdx += 1 + } + return pIdx == pattern.endIndex + } + + @Sendable func test(pattern: UnsafeBufferPointer, candidate: UnsafeBufferPointer) { + let aproximation = RejectionFilter.match( + pattern: RejectionFilter(bytes: pattern), + candidate: RejectionFilter(bytes: candidate) + ) + if aproximation == .no { + XCTAssert(!matches(pattern: pattern, candidate: candidate)) + } + } + + DispatchQueue.concurrentPerform(iterations: Int(UInt8.max)) { pattern0 in + let allBytes = UInt8.min...UInt8.max + let pattern = UnsafeMutablePointer.allocate(capacity: 1) + let candidate = UnsafeMutablePointer.allocate(capacity: 2) + pattern[0] = UInt8(pattern0) + for candidate0 in allBytes { + candidate[0] = candidate0 + test( + pattern: UnsafeBufferPointer(start: pattern, count: 1), + candidate: UnsafeBufferPointer(start: candidate, count: 1) + ) + for candidate1 in allBytes { + candidate[1] = candidate1 + test( + pattern: UnsafeBufferPointer(start: pattern, count: 1), + candidate: UnsafeBufferPointer(start: candidate, count: 2) + ) + } + } + pattern.deallocate() + candidate.deallocate() + } + } +} diff --git a/Tests/CompletionScoringTests/SemanticScoredText.swift b/Tests/CompletionScoringTests/SemanticScoredText.swift new file mode 100644 index 000000000..87e7aa0fb --- /dev/null +++ b/Tests/CompletionScoringTests/SemanticScoredText.swift @@ -0,0 +1,92 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import CompletionScoring +import Foundation + +struct SemanticScoredText { + var text: String + var contentType: Candidate.ContentType + var semanticScore: Double + var groupID: Int? + + init(_ text: String, groupID: Int? = nil, contentType: Candidate.ContentType = .codeCompletionSymbol) { + self.text = text + self.semanticScore = 1.0 + self.groupID = groupID + self.contentType = contentType + } + + init( + _ text: String, + _ semanticScore: Double, + groupID: Int? = nil, + contentType: Candidate.ContentType = .codeCompletionSymbol + ) { + self.text = text + self.semanticScore = semanticScore + self.groupID = groupID + self.contentType = contentType + } + + // Some tests are easier to read in the other order. + init(_ semanticScore: Double, _ text: String, contentType: Candidate.ContentType = .codeCompletionSymbol) { + self.text = text + self.semanticScore = semanticScore + self.contentType = contentType + } + + init( + _ text: String, + _ classification: SemanticClassification, + contentType: Candidate.ContentType = .codeCompletionSymbol + ) { + self.text = text + self.semanticScore = classification.score + self.contentType = contentType + } +} + +extension SemanticClassification { + static func partial( + availability: Availability = .inapplicable, + completionKind: CompletionKind = .other, + flair: Flair = [], + moduleProximity: ModuleProximity = .same, + popularity: Popularity = .none, + scopeProximity: ScopeProximity = .inapplicable, + structuralProximity: StructuralProximity = .project(fileSystemHops: 0), + synchronicityCompatibility: SynchronicityCompatibility = .inapplicable, + typeCompatibility: TypeCompatibility = .inapplicable + ) -> Self { + SemanticClassification( + availability: availability, + completionKind: completionKind, + flair: flair, + moduleProximity: moduleProximity, + popularity: popularity, + scopeProximity: scopeProximity, + structuralProximity: structuralProximity, + synchronicityCompatibility: synchronicityCompatibility, + typeCompatibility: typeCompatibility + ) + } +} + +extension CandidateBatch { + init(candidates: [SemanticScoredText]) { + self.init() + for candidate in candidates { + append(candidate.text, contentType: candidate.contentType) + } + } +} diff --git a/Tests/CompletionScoringTests/StackAllocateTests.swift b/Tests/CompletionScoringTests/StackAllocateTests.swift new file mode 100644 index 000000000..ec6e4c837 --- /dev/null +++ b/Tests/CompletionScoringTests/StackAllocateTests.swift @@ -0,0 +1,106 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import CompletionScoring +import XCTest + +class StackAllocateTests: XCTestCase { + func testAllocating() throws { + UnsafeStackAllocator.withUnsafeStackAllocator { allocator in + let sum3: Int = allocator.withStackArray(of: Int.self, maximumCapacity: 3) { buffer in + buffer.append(2) + buffer.append(4) + buffer.append(6) + return buffer.sum() + } + XCTAssertEqual(sum3, 12) + let sum10240: Int = allocator.withStackArray(of: Int.self, maximumCapacity: 10240) { buffer in + buffer.fill(with: 33) + return buffer.sum() + } + XCTAssertEqual(sum10240, 10240 * 33) + + allocator.withStackArray(of: Int.self, maximumCapacity: 3) { buffer in + XCTAssertEqual(buffer.first, nil) + XCTAssertEqual(buffer.last, nil) + XCTAssertEqual(buffer.sum(), 0) + buffer.append(2) + XCTAssertEqual(buffer.first, 2) + XCTAssertEqual(buffer.last, 2) + XCTAssertEqual(buffer.sum(), 2) + buffer.removeLast() + XCTAssertEqual(buffer.first, nil) + XCTAssertEqual(buffer.last, nil) + XCTAssertEqual(buffer.sum(), 0) + } + } + } + + func testPopLast() throws { + UnsafeStackAllocator.withUnsafeStackAllocator { allocator in + allocator.withStackArray(of: Int.self, maximumCapacity: 1) { buffer in + XCTAssertEqual(nil, buffer.popLast()) + buffer.append(1) + XCTAssertEqual(1, buffer.popLast()) + XCTAssertEqual(0, buffer.count) + XCTAssertEqual(nil, buffer.popLast()) + } + } + } + + func testTruncateTo() throws { + UnsafeStackAllocator.withUnsafeStackAllocator { allocator in + allocator.withStackArray(of: Int.self, maximumCapacity: 2) { buffer in + XCTAssertEqual(buffer.count, 0) + buffer.truncate(to: 0) + XCTAssertEqual(buffer.count, 0) + buffer.truncate(to: 1) + XCTAssertEqual(buffer.count, 0) + + buffer.append(0) + buffer.truncate(to: 1) + XCTAssertEqual(Array(buffer), [0]) + buffer.truncate(to: 0) + XCTAssertEqual(Array(buffer), []) + + buffer.append(0) + buffer.append(1) + buffer.truncate(to: 2) + XCTAssertEqual(Array(buffer), [0, 1]) + buffer.truncate(to: 1) + XCTAssertEqual(Array(buffer), [0]) + buffer.append(1) + buffer.truncate(to: 0) + XCTAssertEqual(Array(buffer), []) + } + } + } + + func testRemoveAll() throws { + UnsafeStackAllocator.withUnsafeStackAllocator { allocator in + allocator.withStackArray(of: Int.self, maximumCapacity: 2) { buffer in + XCTAssertEqual(buffer.count, 0) + buffer.removeAll() + XCTAssertEqual(buffer.count, 0) + buffer.append(0) + XCTAssertEqual(buffer.count, 1) + buffer.removeAll() + XCTAssertEqual(buffer.count, 0) + buffer.append(0) + buffer.append(0) + XCTAssertEqual(buffer.count, 2) + buffer.removeAll() + XCTAssertEqual(buffer.count, 0) + } + } + } +} diff --git a/Tests/CompletionScoringTests/Supporting Files/CodeCompletionFoundationTests-Info.plist b/Tests/CompletionScoringTests/Supporting Files/CodeCompletionFoundationTests-Info.plist new file mode 100644 index 000000000..64d65ca49 --- /dev/null +++ b/Tests/CompletionScoringTests/Supporting Files/CodeCompletionFoundationTests-Info.plist @@ -0,0 +1,22 @@ + + + + + CFBundleDevelopmentRegion + $(DEVELOPMENT_LANGUAGE) + CFBundleExecutable + $(EXECUTABLE_NAME) + CFBundleIdentifier + $(PRODUCT_BUNDLE_IDENTIFIER) + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + $(PRODUCT_NAME) + CFBundlePackageType + $(PRODUCT_BUNDLE_PACKAGE_TYPE) + CFBundleShortVersionString + 1.0 + CFBundleVersion + 1 + + diff --git a/Tests/CompletionScoringTests/SwiftExtensionsTests.swift b/Tests/CompletionScoringTests/SwiftExtensionsTests.swift new file mode 100644 index 000000000..233d0780a --- /dev/null +++ b/Tests/CompletionScoringTests/SwiftExtensionsTests.swift @@ -0,0 +1,161 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import CompletionScoring +import CompletionScoringTestSupport +import Foundation +import XCTest + +class SwiftExtensionsTests: XCTestCase { + func testCompareBytes() { + func compare(_ lhsText: String, _ rhsText: String) -> ComparisonOrder { + Candidate.withAccessToCandidate(for: lhsText, contentType: .codeCompletionSymbol) { lhsCandidate in + Candidate.withAccessToCandidate(for: rhsText, contentType: .codeCompletionSymbol) { rhsCandidate in + compareBytes(lhsCandidate.bytes, rhsCandidate.bytes) + } + } + } + + XCTAssertEqual(compare("", ""), .same) + XCTAssertEqual(compare("", "a"), .ascending) + XCTAssertEqual(compare("a", ""), .descending) + XCTAssertEqual(compare("a", "b"), .ascending) + XCTAssertEqual(compare("b", "a"), .descending) + XCTAssertEqual(compare("a", "a"), .same) + XCTAssertEqual(compare("ab", "ba"), .ascending) + XCTAssertEqual(compare("a", "ba"), .ascending) + XCTAssertEqual(compare("a", "ab"), .ascending) + XCTAssertEqual(compare("ab", "a"), .descending) + XCTAssertEqual(compare("ba", "a"), .descending) + } + + func testConcurrentCompactMap() { + var randomness = RepeatableRandomNumberGenerator() + for _ in 0..<1000 { + let strings = randomness.randomLowercaseASCIIStrings(countRange: 0...100, lengthRange: 0...15) + @Sendable func mappingFunction(_ string: String) -> Int? { + return string.first == "b" ? string.count : nil + } + let concurrentResults = strings.concurrentCompactMap(mappingFunction) + let serialResults = strings.compactMap(mappingFunction) + XCTAssertEqual(concurrentResults, serialResults) + } + } + + func testMinMaxBy() { + func test(min: String?, max: String?, in array: [String]) { + XCTAssertEqual(array.min(by: \.count), min) + XCTAssertEqual(array.max(by: \.count), max) + } + + test(min: "a", max: "a", in: ["a"]) + test(min: "a", max: "a", in: ["a", "b"]) + test(min: "a", max: "bb", in: ["a", "bb"]) + test(min: "a", max: "bb", in: ["bb", "a"]) + test(min: "a", max: "ccc", in: ["a", "bb", "ccc", "dd", "e"]) + test(min: nil, max: nil, in: []) + } + + func testMinMaxOf() { + func test(min: Int?, max: Int?, in array: [String]) { + XCTAssertEqual(array.min(of: \.count), min) + XCTAssertEqual(array.max(of: \.count), max) + } + + test(min: 1, max: 1, in: ["a"]) + test(min: 1, max: 1, in: ["a", "b"]) + test(min: 1, max: 2, in: ["a", "bb"]) + test(min: 1, max: 2, in: ["bb", "a"]) + test(min: 1, max: 3, in: ["a", "bb", "ccc", "dd", "e"]) + test(min: nil, max: nil, in: []) + } + + func testConcurrentMap() { + func test(_ input: [String], transform: @Sendable (String) -> String) { + let serialOutput = input.map(transform) + let concurrentOutput = input.concurrentMap(transform) + let unsafeConcurrentSliceOutput: [String] = input.unsafeSlicedConcurrentMap { (slice, baseAddress) in + for (outputIndex, input) in slice.enumerated() { + baseAddress.advanced(by: outputIndex).initialize(to: transform(input)) + } + } + XCTAssertEqual(serialOutput, concurrentOutput) + XCTAssertEqual(serialOutput, unsafeConcurrentSliceOutput) + } + + let strings = (0..<1000).map { index in + String(repeating: "a", count: index) + } + for stringCount in 0..?) { + needle.withUncachedUTF8Bytes { needle in + body.withUncachedUTF8Bytes { body in + let actualMatch = body.rangeOf(bytes: needle) + XCTAssertEqual(actualMatch, expectedMatch) + } + } + } + search(for: "", in: "", expecting: nil) + search(for: "A", in: "", expecting: nil) + search(for: "A", in: "B", expecting: nil) + search(for: "A", in: "BB", expecting: nil) + search(for: "AA", in: "", expecting: nil) + search(for: "AA", in: "B", expecting: nil) + + search(for: "A", in: "A", expecting: 0 ..+ 1) + search(for: "A", in: "AB", expecting: 0 ..+ 1) + search(for: "A", in: "BA", expecting: 1 ..+ 1) + search(for: "AA", in: "AAB", expecting: 0 ..+ 2) + search(for: "AA", in: "BAA", expecting: 1 ..+ 2) + } + + func testSingleElementBuffer() { + func test(_ value: T) { + var callCount = 0 + UnsafeBufferPointer.withSingleElementBuffer(of: value) { buffer in + callCount += 1 + XCTAssertEqual(buffer.count, 1) + XCTAssertEqual(buffer.first, value) + } + XCTAssertEqual(callCount, 1) + } + test(0) + test(1) + test(0.0) + test(1.0) + test(0..<1) + test(1..<2) + test(UInt8(0)) + test(UInt8(1)) + test("") + test("S") + } +} + +fileprivate struct CompletionItem { + var score: Double + var text: String +} + +extension CompletionItem: Comparable { + static func < (lhs: Self, rhs: Self) -> Bool { + return (lhs.score [Pattern.ScoringWorkload] { + let batches = batchSizes.map { batchSize in + CandidateBatch(symbols: Array(repeating: " ", count: batchSize)) + } + return Pattern.ScoringWorkload.workloads(for: batches, parallelism: parallelism) + } + + func expect(batches: [Int], parallelism: Int, produces: [Pattern.ScoringWorkload]) { + XCTAssertEqual(workloads(for: batches, parallelism: parallelism), produces, "Slicing \(batches)") + } + + /// Shorthand for making slices + func Workload(outputAt: Int, _ units: [Pattern.ScoringWorkload.CandidateBatchSlice]) -> Pattern.ScoringWorkload { + Pattern.ScoringWorkload(outputStartIndex: outputAt, slices: units) + } + + /// Shorthand for making slice units + func Slice(batch: Int, candidates: Range) -> Pattern.ScoringWorkload.CandidateBatchSlice { + return Pattern.ScoringWorkload.CandidateBatchSlice(batchIndex: batch, candidateRange: candidates) + } + + func testLeadingAndTrailingZerosHaveNoImpact() { + // Look for division by zero, never terminating etc with empty cases. + for cores in 1..<3 { + expect(batches: [], parallelism: cores, produces: []) + expect(batches: [0], parallelism: cores, produces: []) + expect(batches: [0, 0], parallelism: cores, produces: []) + + expect( + batches: [1], + parallelism: cores, + produces: [ + Workload(outputAt: 0, [Slice(batch: 0, candidates: 0..<1)]) + ] + ) + expect( + batches: [1, 0], + parallelism: cores, + produces: [ + Workload(outputAt: 0, [Slice(batch: 0, candidates: 0..<1)]) + ] + ) + expect( + batches: [0, 1], + parallelism: cores, + produces: [ + Workload(outputAt: 0, [Slice(batch: 1, candidates: 0..<1)]) + ] + ) + } + } + + func testDivisionsOf2Candidates() throws { + // AA / 1 = AA + expect( + batches: [2], + parallelism: 1, + produces: [ + Workload(outputAt: 0, [Slice(batch: 0, candidates: 0 ..+ 2)]) + ] + ) + + // AA / 2 = A, A + expect( + batches: [2], + parallelism: 2, + produces: [ + Workload(outputAt: 0, [Slice(batch: 0, candidates: 0 ..+ 1)]), + Workload(outputAt: 1, [Slice(batch: 0, candidates: 1 ..+ 1)]), + ] + ) + + // AA / 3 = AA + expect( + batches: [2], + parallelism: 3, + produces: [ + Workload(outputAt: 0, [Slice(batch: 0, candidates: 0 ..+ 2)]) + ] + ) + + // AB / 1 = AB + expect( + batches: [1, 1], + parallelism: 1, + produces: [ + Workload( + outputAt: 0, + [ + Slice(batch: 0, candidates: 0 ..+ 1), + Slice(batch: 1, candidates: 0 ..+ 1), + ] + ) + ] + ) + + // AB / 2 = A, B + expect( + batches: [1, 1], + parallelism: 2, + produces: [ + Workload(outputAt: 0, [Slice(batch: 0, candidates: 0 ..+ 1)]), + Workload(outputAt: 1, [Slice(batch: 1, candidates: 0 ..+ 1)]), + ] + ) + + // AB / 3 = AB + expect( + batches: [1, 1], + parallelism: 3, + produces: [ + Workload( + outputAt: 0, + [ + Slice(batch: 0, candidates: 0 ..+ 1), + Slice(batch: 1, candidates: 0 ..+ 1), + ] + ) + ] + ) + } + + func testDivisonsOf3Candidates() { + // AAA / 1 = AAA + expect( + batches: [3], + parallelism: 1, + produces: [ + Workload(outputAt: 0, [Slice(batch: 0, candidates: 0 ..+ 3)]) + ] + ) + + // AAA / 2 = A, AA + expect( + batches: [3], + parallelism: 2, + produces: [ + Workload(outputAt: 0, [Slice(batch: 0, candidates: 0 ..+ 1)]), + Workload(outputAt: 1, [Slice(batch: 0, candidates: 1 ..+ 2)]), + ] + ) + + // AAA / 3 = A, A, A + expect( + batches: [3], + parallelism: 3, + produces: [ + Workload(outputAt: 0, [Slice(batch: 0, candidates: 0 ..+ 1)]), + Workload(outputAt: 1, [Slice(batch: 0, candidates: 1 ..+ 1)]), + Workload(outputAt: 2, [Slice(batch: 0, candidates: 2 ..+ 1)]), + ] + ) + + // AAA / 4 = AAA + expect( + batches: [3], + parallelism: 4, + produces: [ + Workload(outputAt: 0, [Slice(batch: 0, candidates: 0 ..+ 3)]) + ] + ) + + // ABB / 1 = ABB + expect( + batches: [1, 2], + parallelism: 1, + produces: [ + Workload( + outputAt: 0, + [ + Slice(batch: 0, candidates: 0 ..+ 1), + Slice(batch: 1, candidates: 0 ..+ 2), + ] + ) + ] + ) + + // ABB / 2 = A, BB + expect( + batches: [1, 2], + parallelism: 2, + produces: [ + Workload(outputAt: 0, [Slice(batch: 0, candidates: 0 ..+ 1)]), + Workload(outputAt: 1, [Slice(batch: 1, candidates: 0 ..+ 2)]), + ] + ) + + // ABB / 3 = A, B, B + expect( + batches: [1, 2], + parallelism: 3, + produces: [ + Workload(outputAt: 0, [Slice(batch: 0, candidates: 0 ..+ 1)]), + Workload(outputAt: 1, [Slice(batch: 1, candidates: 0 ..+ 1)]), + Workload(outputAt: 2, [Slice(batch: 1, candidates: 1 ..+ 1)]), + ] + ) + + // ABB / 4 = ABB + expect( + batches: [1, 2], + parallelism: 4, + produces: [ + Workload( + outputAt: 0, + [ + Slice(batch: 0, candidates: 0 ..+ 1), + Slice(batch: 1, candidates: 0 ..+ 2), + ] + ) + ] + ) + + // AAB / 1 = AAB + expect( + batches: [2, 1], + parallelism: 1, + produces: [ + Workload( + outputAt: 0, + [ + Slice(batch: 0, candidates: 0 ..+ 2), + Slice(batch: 1, candidates: 0 ..+ 1), + ] + ) + ] + ) + + // AAB / 2 = A, AB + expect( + batches: [2, 1], + parallelism: 2, + produces: [ + Workload(outputAt: 0, [Slice(batch: 0, candidates: 0 ..+ 1)]), + Workload( + outputAt: 1, + [ + Slice(batch: 0, candidates: 1 ..+ 1), + Slice(batch: 1, candidates: 0 ..+ 1), + ] + ), + ] + ) + + // AAB / 3 = A, A, B + expect( + batches: [2, 1], + parallelism: 3, + produces: [ + Workload(outputAt: 0, [Slice(batch: 0, candidates: 0 ..+ 1)]), + Workload(outputAt: 1, [Slice(batch: 0, candidates: 1 ..+ 1)]), + Workload(outputAt: 2, [Slice(batch: 1, candidates: 0 ..+ 1)]), + ] + ) + + // AAB / 4 = AAB + expect( + batches: [2, 1], + parallelism: 4, + produces: [ + Workload( + outputAt: 0, + [ + Slice(batch: 0, candidates: 0 ..+ 2), + Slice(batch: 1, candidates: 0 ..+ 1), + ] + ) + ] + ) + + // ABC / 1 = ABC + expect( + batches: [1, 1, 1], + parallelism: 1, + produces: [ + Workload( + outputAt: 0, + [ + Slice(batch: 0, candidates: 0 ..+ 1), + Slice(batch: 1, candidates: 0 ..+ 1), + Slice(batch: 2, candidates: 0 ..+ 1), + ] + ) + ] + ) + + // ABC / 2 = A, BC + expect( + batches: [1, 1, 1], + parallelism: 2, + produces: [ + Workload( + outputAt: 0, + [ + Slice(batch: 0, candidates: 0 ..+ 1) + ] + ), + Workload( + outputAt: 1, + [ + Slice(batch: 1, candidates: 0 ..+ 1), + Slice(batch: 2, candidates: 0 ..+ 1), + ] + ), + ] + ) + + // ABC / 3 = A, BC + expect( + batches: [1, 1, 1], + parallelism: 3, + produces: [ + Workload( + outputAt: 0, + [ + Slice(batch: 0, candidates: 0 ..+ 1) + ] + ), + Workload( + outputAt: 1, + [ + Slice(batch: 1, candidates: 0 ..+ 1) + ] + ), + Workload( + outputAt: 2, + [ + Slice(batch: 2, candidates: 0 ..+ 1) + ] + ), + ] + ) + + // ABC / 4 = ABC + expect( + batches: [1, 1, 1], + parallelism: 4, + produces: [ + Workload( + outputAt: 0, + [ + Slice(batch: 0, candidates: 0 ..+ 1), + Slice(batch: 1, candidates: 0 ..+ 1), + Slice(batch: 2, candidates: 0 ..+ 1), + ] + ) + ] + ) + } +} diff --git a/Tests/CompletionScoringTests/TestTimings.swift b/Tests/CompletionScoringTests/TestTimings.swift new file mode 100644 index 000000000..26d18061a --- /dev/null +++ b/Tests/CompletionScoringTests/TestTimings.swift @@ -0,0 +1,71 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import CompletionScoring +import CompletionScoringTestSupport +import XCTest + +class TestTimings: XCTestCase { + func test(_ values: [Double], _ accessor: (Timings) -> R, _ expected: R) { + let actual = accessor(Timings(values)) + XCTAssertEqual(actual, expected) + } + + func test(_ values: [Double], _ accessor: (Timings) -> R, _ expected: Range) { + let actual = accessor(Timings(values)) + XCTAssert(expected.contains(actual)) + } + + func testMin() { + test([], \.stats?.min, nil) + test([4], \.stats?.min, 4) + test([4, 2], \.stats?.min, 2) + test([4, 2, 9], \.stats?.min, 2) + } + + func testAverage() { + test([], \.stats?.min, nil) + test([4], \.stats?.min, 4) + test([4, 2], \.stats?.min, 2) + test([4, 2, 9], \.stats?.min, 2) + } + + func testMax() { + test([], \.stats?.max, nil) + test([4], \.stats?.max, 4) + test([4, 2], \.stats?.max, 4) + test([4, 2, 9], \.stats?.max, 9) + } + + func testAverageDeviation() { + test([], \.meanAverageDeviation, 0) + test([2], \.meanAverageDeviation, 0) + test([2, 2], \.meanAverageDeviation, 0) + test([2, 4], \.meanAverageDeviation, 1) + } + + func testStandardDeviation() { + test([], \.standardDeviation, 0) + test([2], \.standardDeviation, 0) + test([2, 2], \.standardDeviation, 0) + test([2, 4, 6, 2, 6], \.standardDeviation, 2) + test([1, 2, 3, 4, 5], \.standardDeviation, 1.5811..<1.5812) + } + + func testStandardError() { + test([], \.standardError, 0) + test([2], \.standardError, 0) + test([2, 2], \.standardError, 0) + test([1, 2, 3, 4, 5], \.standardError, 0.70710..<0.70711) + test([1, 2, 4, 8, 16], \.standardError, 2.72763..<2.72764) + } +} diff --git a/Tests/CompletionScoringTests/TopKTests.swift b/Tests/CompletionScoringTests/TopKTests.swift new file mode 100644 index 000000000..45f5694b3 --- /dev/null +++ b/Tests/CompletionScoringTests/TopKTests.swift @@ -0,0 +1,76 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import CompletionScoring +import XCTest + +class TopKTests: XCTestCase { + func testSelectTopK() throws { + func select(k: Int, from: [Int]) -> [Int] { + from.fastTopK(k, lessThan: <) + } + + func test(_ all: [Int], _ expected: [Int]) { + XCTAssertEqual(all.fastTopK(expected.count, lessThan: <).sorted(by: <), expected.sorted(by: <)) + } + + test([], []) + test([1], [1]) + test([1, 2], [1]) + test([2, 1], [1]) + test([1, 2, 3], [1]) + test([1, 2, 3], [1, 2]) + test([1, 2, 3], [1, 2, 3]) + test([3, 2, 1], [1]) + test([3, 2, 1], [1, 2]) + test([3, 2, 1], [1, 2, 3]) + } + + func testSelectTopKExhaustively() throws { + func allCombinations(count: Int, body: ([Int]) -> ()) { + var array = [Int](repeating: 0, count: count) + func enumerate(slot: Int) { + if slot == array.count { + body(array) + } else { + for x in 0.. Bool) -> [Element] { + var copy = UnsafeMutableBufferPointer.allocate(copyOf: self); defer { copy.deinitializeAllAndDeallocate() } + copy.selectTopKAndTruncate(k, lessThan: lessThan) + return Array(copy) + } + + func slowTopK(_ k: Int, lessThan: (Element, Element) -> Bool) -> Self { + Array(sorted(by: lessThan).prefix(k)) + } +} diff --git a/Tests/CompletionScoringTests/UTF8ByteTests.swift b/Tests/CompletionScoringTests/UTF8ByteTests.swift new file mode 100644 index 000000000..a8dd39ae6 --- /dev/null +++ b/Tests/CompletionScoringTests/UTF8ByteTests.swift @@ -0,0 +1,60 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import CompletionScoring +import Foundation +import XCTest + +class UTF8ByteTests: XCTestCase { + func testCaseMapping() { + func byte(_ character: Character) -> UTF8Byte { + character.utf8.only! + } + struct BoundaryLetters { + var lowercase: UTF8Byte + var uppercase: UTF8Byte + } + + let boundaryLetters: [BoundaryLetters] = [ + BoundaryLetters(lowercase: byte("a"), uppercase: byte("A")), + BoundaryLetters(lowercase: byte("b"), uppercase: byte("B")), + BoundaryLetters(lowercase: byte("y"), uppercase: byte("Y")), + BoundaryLetters(lowercase: byte("z"), uppercase: byte("Z")), + ] + + for boundaryLetter in boundaryLetters { + XCTAssertTrue(boundaryLetter.uppercase.isUppercase) + XCTAssertTrue(boundaryLetter.lowercase.isLowercase) + XCTAssertTrue(!boundaryLetter.uppercase.isLowercase) + XCTAssertTrue(!boundaryLetter.lowercase.isUppercase) + + XCTAssertEqual(boundaryLetter.uppercase.uppercasedUTF8Byte, boundaryLetter.uppercase) + XCTAssertEqual(boundaryLetter.lowercase.uppercasedUTF8Byte, boundaryLetter.uppercase) + + XCTAssertEqual(boundaryLetter.uppercase.lowercasedUTF8Byte, boundaryLetter.lowercase) + XCTAssertEqual(boundaryLetter.lowercase.lowercasedUTF8Byte, boundaryLetter.lowercase) + } + + let boundarySymbols: [UTF8Byte] = [ + byte("A") - 1, + byte("Z") + 1, + byte("a") - 1, + byte("z") + 1, + ] + for boundarySymbol in boundarySymbols { + XCTAssertTrue(!boundarySymbol.isUppercase) + XCTAssertTrue(!boundarySymbol.isLowercase) + XCTAssertEqual(boundarySymbol.uppercasedUTF8Byte, boundarySymbol) + XCTAssertEqual(boundarySymbol.lowercasedUTF8Byte, boundarySymbol) + } + } +} diff --git a/Tests/CompletionScoringTests/XCTestCaseScoringAdditions.swift b/Tests/CompletionScoringTests/XCTestCaseScoringAdditions.swift new file mode 100644 index 000000000..a64cf0022 --- /dev/null +++ b/Tests/CompletionScoringTests/XCTestCaseScoringAdditions.swift @@ -0,0 +1,138 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import CompletionScoring +import XCTest + +extension XCTestCase { + func score( + patternText: String, + candidate: SemanticScoredText, + contenType: Pattern.ContentType = .codeCompletionSymbol, + precision: Pattern.Precision + ) -> CompletionScore { + let textScore = Candidate.withAccessToCandidate(for: candidate.text, contentType: contenType) { candidate in + Pattern(text: patternText).score(candidate: candidate, precision: precision) + } + return CompletionScore(textComponent: textScore, semanticComponent: candidate.semanticScore) + } + + func test( + _ patternText: String, + precision: Pattern.Precision, + prefers expectedWinner: SemanticScoredText, + over expectedLoser: SemanticScoredText, + contentType: Pattern.ContentType = .codeCompletionSymbol + ) { + let expectedWinnerScore = score( + patternText: patternText, + candidate: expectedWinner, + contenType: contentType, + precision: precision + ) + let expectedLoserScore = score( + patternText: patternText, + candidate: expectedLoser, + contenType: contentType, + precision: precision + ) + func failureMessage() -> String { + formatTable(rows: [ + ["Expected Winner", "Semantic Scores", "Text Score", "Composite Score", "Candidate Text"], + [ + "✓", expectedWinnerScore.semanticComponent.format(precision: 3), + expectedWinnerScore.textComponent.format(precision: 3), + expectedWinnerScore.value.format(precision: 3), expectedWinner.text, + ], + [ + "", expectedLoserScore.semanticComponent.format(precision: 3), + expectedLoserScore.textComponent.format(precision: 3), + expectedLoserScore.value.format(precision: 3), expectedLoser.text, + ], + ]) + } + XCTAssert( + expectedWinnerScore > expectedLoserScore, + "\"\(patternText)\" should match \"\(expectedWinner.text)\" better than \"\(expectedLoser.text)\".\n\(failureMessage())\n" + ) + } + + func score( + patternText: String, + candidateText: String, + contenType: Pattern.ContentType = .codeCompletionSymbol, + precision: Pattern.Precision + ) -> Double { + score(patternText: patternText, candidate: SemanticScoredText(candidateText), precision: precision) + .textComponent + } + + func test( + _ patternText: String, + precision: Pattern.Precision, + prefers expectedWinnerText: String, + over expectedLoserText: String + ) { + test( + patternText, + precision: precision, + prefers: SemanticScoredText(expectedWinnerText), + over: SemanticScoredText(expectedLoserText) + ) + } +} + +extension String { + func enumeratePrefixes(includeLowercased: Bool, body: (String) -> ()) { + for length in 1.. String { + String(format: "%.0\(precision)f", self) + } +} + +fileprivate func formatTable(rows: [[String]]) -> String { + if let headers = rows.first { + let separator = " | " + let headerSeparatorColumnSeparator = "-+-" + var columnWidths = Array(repeating: 0, count: headers.count) + for row in rows { + precondition(row.count == headers.count) + for (columnIndex, cell) in row.enumerated() { + columnWidths[columnIndex] = max(columnWidths[columnIndex], cell.count) + } + } + var formatedRows: [String] = rows.map { row in + let indentedCells: [String] = row.enumerated().map { columnIndex, cell in + let spacesToAdd = columnWidths[columnIndex] - cell.count + let indent = String(repeating: " ", count: spacesToAdd) + return indent + cell + } + return indentedCells.joined(separator: separator) + } + let headerSeparator = headers.enumerated().map { (columnIndex, _) in + String(repeating: "-", count: columnWidths[columnIndex]) + }.joined(separator: headerSeparatorColumnSeparator) + formatedRows.insert(headerSeparator, at: 1) + return formatedRows.joined(separator: "\n") + } else { + return "" + } +} diff --git a/Tests/SourceKitDTests/SourceKitDRegistryTests.swift b/Tests/SourceKitDTests/SourceKitDRegistryTests.swift index b25660b0f..c98f4a9b6 100644 --- a/Tests/SourceKitDTests/SourceKitDRegistryTests.swift +++ b/Tests/SourceKitDTests/SourceKitDRegistryTests.swift @@ -66,6 +66,9 @@ private let nextToken = AtomicUInt32(initialValue: 0) final class FakeSourceKitD: SourceKitD { let token: UInt32 var api: sourcekitd_api_functions_t { fatalError() } + var ideApi: sourcekitd_ide_api_functions_t { fatalError() } + var pluginApi: sourcekitd_plugin_api_functions_t { fatalError() } + var servicePluginApi: sourcekitd_service_plugin_api_functions_t { fatalError() } var keys: sourcekitd_api_keys { fatalError() } var requests: sourcekitd_api_requests { fatalError() } var values: sourcekitd_api_values { fatalError() } @@ -76,7 +79,7 @@ final class FakeSourceKitD: SourceKitD { } static func getOrCreate(_ url: URL, in registry: SourceKitDRegistry) async -> SourceKitD { - return await registry.getOrAdd(url, create: { Self.init() }) + return await registry.getOrAdd(url, pluginPaths: nil, create: { Self.init() }) } package func log(request: SKDRequestDictionary) {} diff --git a/Tests/SwiftSourceKitPluginTests/PluginPaths.swift b/Tests/SwiftSourceKitPluginTests/PluginPaths.swift new file mode 100644 index 000000000..fa13488b5 --- /dev/null +++ b/Tests/SwiftSourceKitPluginTests/PluginPaths.swift @@ -0,0 +1,107 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation +import SourceKitD + +/// The path to the `SwiftSourceKitPluginTests` test bundle. This gives us a hook into the the build directory. +private let xctestBundle: URL = { + #if canImport(Darwin) + for bundle in Bundle.allBundles where bundle.bundlePath.hasSuffix(".xctest") { + return bundle.bundleURL + } + preconditionFailure("Failed to find xctest bundle") + #else + return URL( + fileURLWithPath: CommandLine.arguments.first!, + relativeTo: URL(fileURLWithPath: FileManager.default.currentDirectoryPath) + ) + #endif +}() + +/// When running tests from Xcode, determine the build configuration of the package. +var inferedXcodeBuildConfiguration: String? { + if let xcodeBuildDirectory = ProcessInfo.processInfo.environment["__XCODE_BUILT_PRODUCTS_DIR_PATHS"] { + return URL(fileURLWithPath: xcodeBuildDirectory).lastPathComponent + } else { + return nil + } +} + +/// Shorthand for `FileManager.fileExists` +private func fileExists(at url: URL) -> Bool { + return FileManager.default.fileExists(atPath: url.path) +} + +/// Try to find the client and server plugin relative to `base`. +/// +/// Implementation detail of `sourceKitPluginPaths` which walks up the directory structure, repeatedly calling this method. +private func pluginPaths(relativeTo base: URL) -> PluginPaths? { + // When building in Xcode + if let buildConfiguration = inferedXcodeBuildConfiguration { + let frameworksDir = base.appendingPathComponent("Products") + .appendingPathComponent(buildConfiguration) + .appendingPathComponent("PackageFrameworks") + let clientPlugin = + frameworksDir + .appendingPathComponent("SwiftSourceKitClientPlugin.framework") + .appendingPathComponent("SwiftSourceKitClientPlugin") + let servicePlugin = + frameworksDir + .appendingPathComponent("SwiftSourceKitPlugin.framework") + .appendingPathComponent("SwiftSourceKitPlugin") + if fileExists(at: clientPlugin) && fileExists(at: servicePlugin) { + return PluginPaths(clientPlugin: clientPlugin, servicePlugin: servicePlugin) + } + } + + // When creating an `xctestproducts` bundle + do { + let frameworksDir = base.appendingPathComponent("PackageFrameworks") + let clientPlugin = + frameworksDir + .appendingPathComponent("SwiftSourceKitClientPlugin.framework") + .appendingPathComponent("SwiftSourceKitClientPlugin") + let servicePlugin = + frameworksDir + .appendingPathComponent("SwiftSourceKitPlugin.framework") + .appendingPathComponent("SwiftSourceKitPlugin") + if fileExists(at: clientPlugin) && fileExists(at: servicePlugin) { + return PluginPaths(clientPlugin: clientPlugin, servicePlugin: servicePlugin) + } + } + + // When building using 'swift test' + do { + let clientPlugin = base.appendingPathComponent("libSwiftSourceKitClientPlugin.dylib") + let servicePlugin = base.appendingPathComponent("libSwiftSourceKitPlugin.dylib") + if fileExists(at: clientPlugin) && fileExists(at: servicePlugin) { + return PluginPaths(clientPlugin: clientPlugin, servicePlugin: servicePlugin) + } + } + + return nil +} + +/// Returns the path the the client plugin and the server plugin within the current build directory. +/// +/// Returns `nil` if either of the plugins can't be found in the build directory. +let sourceKitPluginPaths: PluginPaths? = { + var base = xctestBundle + while base.pathComponents.count > 1 { + if let paths = pluginPaths(relativeTo: base) { + return paths + } + base = base.deletingLastPathComponent() + } + return nil +}() diff --git a/Tests/SwiftSourceKitPluginTests/PluginSwiftPMTestProject.swift b/Tests/SwiftSourceKitPluginTests/PluginSwiftPMTestProject.swift new file mode 100644 index 000000000..15e15ed98 --- /dev/null +++ b/Tests/SwiftSourceKitPluginTests/PluginSwiftPMTestProject.swift @@ -0,0 +1,114 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import BuildSystemIntegration +import Foundation +import LanguageServerProtocol +import SKTestSupport +import ToolchainRegistry + +/// A SwiftPM project that is built from files specified inside the test case and that can provide compiler arguments +/// for those files. +/// +/// Compared to `SwiftPMTestProject`, the main difference is that this does not start a SourceKit-LSP server. +final class PluginSwiftPMTestProject { + /// The directory in which the temporary files are being placed. + let scratchDirectory: URL + + private let fileData: [String: MultiFileTestProject.FileData] + + private var _buildSystemManager: BuildSystemManager? + private var buildSystemManager: BuildSystemManager { + get async { + if let _buildSystemManager { + return _buildSystemManager + } + let buildSystemManager = await BuildSystemManager( + buildSystemSpec: BuildSystemSpec(kind: .swiftPM, projectRoot: scratchDirectory), + toolchainRegistry: .forTesting, + options: .testDefault(backgroundIndexing: false), + connectionToClient: DummyBuildSystemManagerConnectionToClient(), + buildSystemTestHooks: BuildSystemTestHooks() + ) + _buildSystemManager = buildSystemManager + return buildSystemManager + } + } + + enum Error: Swift.Error { + /// No file with the given filename is known to the `PluginSwiftPMTestProject`. + case fileNotFound + + /// `PluginSwiftPMTestProject` did not produce compiler arguments for a file. + case failedToRetrieveCompilerArguments + } + + package init( + files: [RelativeFileLocation: String], + testName: String = #function + ) async throws { + scratchDirectory = try testScratchDir(testName: testName) + self.fileData = try MultiFileTestProject.writeFilesToDisk(files: files, scratchDirectory: scratchDirectory) + + // Build package + try await SwiftPMTestProject.build(at: scratchDirectory) + } + + deinit { + if cleanScratchDirectories { + try? FileManager.default.removeItem(at: scratchDirectory) + } + } + + /// Returns the URI of the file with the given name. + package func uri(for fileName: String) throws -> DocumentURI { + guard let fileData = self.fileData[fileName] else { + throw Error.fileNotFound + } + return fileData.uri + } + + /// Returns the position of the given marker in the given file. + package func position(of marker: String, in fileName: String) throws -> Position { + guard let fileData = self.fileData[fileName] else { + throw Error.fileNotFound + } + return DocumentPositions(markedText: fileData.markedText)[marker] + } + + /// Returns the contents of the file with the given name. + package func contents(of fileName: String) throws -> String { + guard let fileData = self.fileData[fileName] else { + throw Error.fileNotFound + } + return extractMarkers(fileData.markedText).textWithoutMarkers + } + + package func compilerArguments(for fileName: String) async throws -> [String] { + await buildSystemManager.waitForUpToDateBuildGraph() + let buildSettings = await buildSystemManager.buildSettingsInferredFromMainFile( + for: try uri(for: fileName), + language: .swift, + fallbackAfterTimeout: false + ) + guard let buildSettings else { + throw Error.failedToRetrieveCompilerArguments + } + let compilerArguments = buildSettings.compilerArguments + if compilerArguments.first?.hasSuffix("swiftc") ?? false { + // Compiler arguments returned from SwiftPMWorkspace contain the compiler executable. + // sourcekitd does not expect the compiler arguments to contain the executable. + return Array(compilerArguments.dropFirst()) + } + return compilerArguments + } +} diff --git a/Tests/SwiftSourceKitPluginTests/SwiftSourceKitPluginTests.swift b/Tests/SwiftSourceKitPluginTests/SwiftSourceKitPluginTests.swift new file mode 100644 index 000000000..1645c73e7 --- /dev/null +++ b/Tests/SwiftSourceKitPluginTests/SwiftSourceKitPluginTests.swift @@ -0,0 +1,2061 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import CompletionScoring +import Csourcekitd +import LanguageServerProtocol +import SKTestSupport +import SourceKitD +import SwiftExtensions +import ToolchainRegistry +import XCTest + +final class SwiftSourceKitPluginTests: XCTestCase { + /// Returns the paths from which the SourceKit plugins should be loaded or throws an error if the plugins cannot be + /// found. + private var pluginPaths: PluginPaths { + get throws { + struct PluginLoadingError: Error, CustomStringConvertible { + var description: String = "Could not find SourceKit plugin" + } + + guard let sourceKitPluginPaths else { + // If we couldn't find the plugins, keep `didLoadPlugin = false`, which will throw an error in each test case's + // `setUp` function. We can't throw an error from the class `setUp` function. + throw PluginLoadingError() + } + return sourceKitPluginPaths + } + } + + /// Returns a path to a file name that is unique to this test execution. + /// + /// The file does not actually exist on disk. + private func scratchFilePath(testName: String = #function, fileName: String = "a.swift") -> String { + return "/\(testScratchName(testName: testName))/\(fileName)" + } + + func getSourceKitD() async throws -> SourceKitD { + guard let sourcekitd = await ToolchainRegistry.forTesting.default?.sourcekitd else { + struct NoSourceKitdFound: Error, CustomStringConvertible { + var description: String = "Could not find SourceKitD" + } + throw NoSourceKitdFound() + } + return try await DynamicallyLoadedSourceKitD.getOrCreate(dylibPath: sourcekitd, pluginPaths: try self.pluginPaths) + } + + func testBasicCompletion() async throws { + let sourcekitd = try await getSourceKitD() + let path = scratchFilePath() + let positions = try await sourcekitd.openDocument( + path, + contents: """ + struct S { + func test() { + self.1️⃣ 2️⃣ + } + } + """, + compilerArguments: [path] + ) + + await assertThrowsError( + try await sourcekitd.completeUpdate(path: path, position: positions["1️⃣"], filter: ""), + expectedMessage: #/no matching session/# + ) + + await assertThrowsError( + try await sourcekitd.completeClose(path: path, position: positions["1️⃣"]), + expectedMessage: #/no matching session/# + ) + + func checkTestMethod(result: CompletionResultSet, file: StaticString = #filePath, line: UInt = #line) { + guard let test = result.items.first(where: { $0.name == "test()" }) else { + XCTFail("did not find test(); got \(result.items)", file: file, line: line) + return + } + XCTAssertEqual(test.kind, sourcekitd.values.declMethodInstance, file: file, line: line) + XCTAssertEqual(test.description, "test()", file: file, line: line) + XCTAssertEqual(test.sourcetext, "test()", file: file, line: line) + XCTAssertEqual(test.typename, "Void", file: file, line: line) + XCTAssertEqual(test.priorityBucket, 9, file: file, line: line) + XCTAssertFalse(test.semanticScore.isNaN, file: file, line: line) + XCTAssertEqual(test.isSystem, false, file: file, line: line) + XCTAssertEqual(test.numBytesToErase, 0, file: file, line: line) + XCTAssertEqual(test.hasDiagnostic, false, file: file, line: line) + } + + func checkTestMethodAnnotated(result: CompletionResultSet, file: StaticString = #filePath, line: UInt = #line) { + guard let test = result.items.first(where: { $0.name == "test()" }) else { + XCTFail("did not find test(); got \(result.items)", file: file, line: line) + return + } + XCTAssertEqual(test.kind, sourcekitd.values.declMethodInstance, file: file, line: line) + XCTAssertEqual(test.description, "test()", file: file, line: line) + XCTAssertEqual(test.sourcetext, "test()", file: file, line: line) + XCTAssertEqual(test.typename, "Void", file: file, line: line) + XCTAssertEqual(test.priorityBucket, 9, file: file, line: line) + XCTAssertFalse(test.semanticScore.isNaN, file: file, line: line) + XCTAssertEqual(test.isSystem, false, file: file, line: line) + XCTAssertEqual(test.numBytesToErase, 0, file: file, line: line) + XCTAssertEqual(test.hasDiagnostic, false, file: file, line: line) + } + + var unfilteredResultCount: Int? = nil + + let result1 = try await sourcekitd.completeOpen(path: path, position: positions["1️⃣"], filter: "") + XCTAssertEqual(result1.items.count, result1.unfilteredResultCount) + checkTestMethod(result: result1) + unfilteredResultCount = result1.unfilteredResultCount + + let result2 = try await sourcekitd.completeUpdate(path: path, position: positions["1️⃣"], filter: "") + XCTAssertEqual(result2.items.count, result2.unfilteredResultCount) + XCTAssertEqual(result2.unfilteredResultCount, unfilteredResultCount) + checkTestMethod(result: result2) + + let result3 = try await sourcekitd.completeUpdate(path: path, position: positions["1️⃣"], filter: "test") + XCTAssertEqual(result3.unfilteredResultCount, unfilteredResultCount) + XCTAssertEqual(result3.items.count, 1) + checkTestMethod(result: result3) + + let result4 = try await sourcekitd.completeUpdate(path: path, position: positions["1️⃣"], filter: "testify") + XCTAssertEqual(result4.unfilteredResultCount, unfilteredResultCount) + XCTAssertEqual(result4.items.count, 0) + + // Update on different location + await assertThrowsError( + try await sourcekitd.completeUpdate(path: path, position: positions["2️⃣"], filter: ""), + expectedMessage: #/no matching session/# + ) + await assertThrowsError( + try await sourcekitd.completeClose(path: path, position: positions["2️⃣"]), + expectedMessage: #/no matching session/# + ) + + // Update on different location + await assertThrowsError( + try await sourcekitd.completeUpdate(path: "/other.swift", position: positions["1️⃣"], filter: ""), + expectedMessage: #/no matching session/# + ) + + // Annotated + let result5 = try await sourcekitd.completeOpen( + path: path, + position: positions["1️⃣"], + filter: "", + flags: [.annotate] + ) + XCTAssertEqual(result5.items.count, result5.unfilteredResultCount) + checkTestMethodAnnotated(result: result5) + + let result6 = try await sourcekitd.completeUpdate( + path: path, + position: positions["1️⃣"], + filter: "test", + flags: [.annotate] + ) + XCTAssertEqual(result6.items.count, 1) + checkTestMethodAnnotated(result: result6) + + try await sourcekitd.completeClose(path: path, position: positions["1️⃣"]) + + await assertThrowsError( + try await sourcekitd.completeUpdate(path: path, position: positions["1️⃣"], filter: ""), + expectedMessage: #/no matching session/# + ) + } + + func testEmptyName() async throws { + let sourcekitd = try await getSourceKitD() + let path = scratchFilePath() + let positions = try await sourcekitd.openDocument( + path, + contents: """ + func test1(closure: () -> Void) { + closure(1️⃣ + } + func noArg() -> String {} + func noArg() -> Int {} + func test2() { + noArg(2️⃣ + } + """, + compilerArguments: [path] + ) + let result1 = try await sourcekitd.completeOpen( + path: path, + position: positions["1️⃣"], + filter: "", + flags: [.annotate] + ) + XCTAssertEqual(result1.items.count, 1) + XCTAssertEqual(result1.items[0].name, "") + + let result2 = try await sourcekitd.completeOpen( + path: path, + position: positions["2️⃣"], + filter: "", + flags: [.annotate], + maxResults: 1 + ) + XCTAssertEqual(result2.items.count, 1) + XCTAssertEqual(result2.items[0].name, "") + let doc = try await sourcekitd.completeDocumentation(id: result2.items[0].id) + XCTAssertEqual(doc.docBrief, nil) + } + + func testMultipleFiles() async throws { + let sourcekitd = try await getSourceKitD() + let pathA = scratchFilePath(fileName: "a.swift") + let pathB = scratchFilePath(fileName: "b.swift") + let positionsA = try await sourcekitd.openDocument( + pathA, + contents: """ + struct A { + func aaa(b: B) { + b.1️⃣ + } + } + """, + compilerArguments: [pathA, pathB] + ) + let positionsB = try await sourcekitd.openDocument( + pathB, + contents: """ + struct B { + func bbb(a: A) { + a.2️⃣ + } + } + """, + compilerArguments: [pathA, pathB] + ) + + func checkResult(name: String, result: CompletionResultSet, file: StaticString = #filePath, line: UInt = #line) { + guard let test = result.items.first(where: { $0.name == name }) else { + XCTFail("did not find \(name); got \(result.items)", file: file, line: line) + return + } + XCTAssertEqual(test.kind, sourcekitd.values.declMethodInstance, file: file, line: line) + } + + let result1 = try await sourcekitd.completeOpen( + path: pathA, + position: positionsA["1️⃣"], + filter: "" + ) + checkResult(name: "bbb(a:)", result: result1) + + let result2 = try await sourcekitd.completeUpdate( + path: pathA, + position: positionsA["1️⃣"], + filter: "b" + ) + checkResult(name: "bbb(a:)", result: result2) + + let result3 = try await sourcekitd.completeOpen( + path: pathB, + position: positionsB["2️⃣"], + filter: "" + ) + checkResult(name: "aaa(b:)", result: result3) + + let result4 = try await sourcekitd.completeUpdate( + path: pathB, + position: positionsB["2️⃣"], + filter: "a" + ) + checkResult(name: "aaa(b:)", result: result4) + } + + func testCancellation() async throws { + let sourcekitd = try await getSourceKitD() + let path = scratchFilePath() + let positions = try await sourcekitd.openDocument( + path, + contents: """ + struct A: ExpressibleByIntegerLiteral { init(integerLiteral value: Int) {} } + struct B: ExpressibleByIntegerLiteral { init(integerLiteral value: Int) {} } + struct C: ExpressibleByIntegerLiteral { init(integerLiteral value: Int) {} } + + func + (lhs: A, rhs: B) -> A { fatalError() } + func + (lhs: B, rhs: C) -> A { fatalError() } + func + (lhs: C, rhs: A) -> A { fatalError() } + + func + (lhs: B, rhs: A) -> B { fatalError() } + func + (lhs: C, rhs: B) -> B { fatalError() } + func + (lhs: A, rhs: C) -> B { fatalError() } + + func + (lhs: C, rhs: B) -> C { fatalError() } + func + (lhs: B, rhs: C) -> C { fatalError() } + func + (lhs: A, rhs: A) -> C { fatalError() } + + class Foo { + func slow(x: Invalid1, y: Invalid2) { + let x: C = 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + 1️⃣ + } + + struct Foo { + let fooMember: String + } + + func fast(a: Foo) { + a.2️⃣ + } + } + """, + compilerArguments: [path] + ) + + let slowCompletionResultReceived = self.expectation(description: "slow completion") + let slowCompletionTask = Task { + do { + _ = try await sourcekitd.completeOpen( + path: path, + position: positions["1️⃣"], + filter: "" + ) + XCTFail("Expected completion to be cancelled") + } catch { + XCTAssert(error is CancellationError, "Expected completion to be cancelled, failed with \(error)") + } + slowCompletionResultReceived.fulfill() + } + slowCompletionTask.cancel() + try await fulfillmentOfOrThrow([slowCompletionResultReceived], timeout: 30) + + let fastCompletionStarted = Date() + let result = try await sourcekitd.completeOpen( + path: path, + position: positions["2️⃣"], + filter: "" + ) + XCTAssert(result.items.count > 0) + XCTAssertLessThan(Date().timeIntervalSince(fastCompletionStarted), 30) + } + + func testEdits() async throws { + let sourcekitd = try await getSourceKitD() + let path = scratchFilePath() + let positions = try await sourcekitd.openDocument( + path, + contents: """ + struct S { + func test() { + let solf = 1 + solo.1️⃣ + } + func magic_method_of_greatness() {} + } + """, + compilerArguments: [path] + ) + + let result1 = try await sourcekitd.completeOpen( + path: path, + position: positions["1️⃣"], + filter: "magic_method_of_greatness", + flags: [] + ) + XCTAssertEqual(result1.unfilteredResultCount, 0) + XCTAssertEqual(result1.items.count, 0) + + let sOffset = """ + struct S { + func test() { + let solo = 1 + s + """.count - 1 + + try await sourcekitd.editDocument(path, fromOffset: sOffset + 1, length: 1, newContents: "e") + try await sourcekitd.editDocument(path, fromOffset: sOffset + 3, length: 1, newContents: "f") + + let result2 = try await sourcekitd.completeOpen( + path: path, + position: positions["1️⃣"], + filter: "magic_method_of_greatness", + flags: [] + ) + XCTAssertGreaterThan(result2.unfilteredResultCount, 1) + XCTAssertEqual(result2.items.count, 1) + + try await sourcekitd.editDocument(path, fromOffset: sOffset, length: 3, newContents: "") + try await sourcekitd.editDocument(path, fromOffset: sOffset, length: 0, newContents: "sel") + + let result3 = try await sourcekitd.completeOpen( + path: path, + position: positions["1️⃣"], + filter: "magic_method_of_greatness", + flags: [] + ) + XCTAssertGreaterThan(result3.unfilteredResultCount, 1) + XCTAssertEqual(result3.items.count, 1) + } + + func testDocumentation() async throws { + let sourcekitd = try await getSourceKitD() + let path = scratchFilePath() + let positions = try await sourcekitd.openDocument( + path, + contents: """ + protocol P { + /// Protocol P foo1 + func foo1() + } + struct S: P { + func foo1() {} + /// Struct S foo2 + func foo2() {} + func foo3() {} + func test() { + self.1️⃣ + } + } + """, + compilerArguments: [path] + ) + + let result = try await sourcekitd.completeOpen( + path: path, + position: positions["1️⃣"], + filter: "foo" + ) + XCTAssertGreaterThan(result.unfilteredResultCount, 3) + let sym1 = try unwrap(result.items.first(where: { $0.name == "foo1()" }), "did not find foo1; got \(result.items)") + let sym2 = try unwrap(result.items.first(where: { $0.name == "foo2()" }), "did not find foo2; got \(result.items)") + let sym3 = try unwrap(result.items.first(where: { $0.name == "foo3()" }), "did not find foo3; got \(result.items)") + + let sym1Doc = try await sourcekitd.completeDocumentation(id: sym1.id) + XCTAssertEqual(sym1Doc.docBrief, "Protocol P foo1") + XCTAssertEqual(sym1Doc.associatedUSRs, ["s:1a1SV4foo1yyF", "s:1a1PP4foo1yyF"]) + + let sym2Doc = try await sourcekitd.completeDocumentation(id: sym2.id) + XCTAssertEqual(sym2Doc.docBrief, "Struct S foo2") + XCTAssertEqual(sym2Doc.associatedUSRs, ["s:1a1SV4foo2yyF"]) + + let sym3Doc = try await sourcekitd.completeDocumentation(id: sym3.id) + XCTAssertNil(sym3Doc.docBrief) + XCTAssertEqual(sym3Doc.associatedUSRs, ["s:1a1SV4foo3yyF"]) + } + + func testNumBytesToErase() async throws { + let sourcekitd = try await getSourceKitD() + let path = scratchFilePath() + let positions = try await sourcekitd.openDocument( + path, + contents: """ + struct S { var myVar: Int } + func test(s: S?) { + s.1️⃣ + } + """, + compilerArguments: [path] + ) + + let result = try await sourcekitd.completeOpen( + path: path, + position: positions["1️⃣"], + filter: "" + ) + XCTAssertEqual(result.items.count, result.unfilteredResultCount) + let myVar = try unwrap(result.items.first(where: { $0.name == "myVar" }), "did not find myVar; got \(result.items)") + XCTAssertEqual(myVar.isSystem, false) + XCTAssertEqual(myVar.numBytesToErase, 1) + + let unwrapped = try unwrap( + result.items.first(where: { $0.name == "unsafelyUnwrapped" }), + "did not find myVar; got \(result.items)" + ) + + XCTAssertEqual(unwrapped.isSystem, true) + XCTAssertEqual(unwrapped.numBytesToErase, 0) + } + + func testObjectLiterals() async throws { + let sourcekitd = try await getSourceKitD() + let path = scratchFilePath() + let positions = try await sourcekitd.openDocument( + path, + contents: """ + func test() { + }1️⃣ + """, + compilerArguments: [path] + ) + + let result = try await sourcekitd.completeOpen( + path: path, + position: positions["1️⃣"], + filter: "" + ) + XCTAssertFalse( + result.items.contains(where: { + $0.description.hasPrefix("#colorLiteral") || $0.description.hasPrefix("#imageLiteral") + }) + ) + } + + func testAddInitsToTopLevel() async throws { + let sourcekitd = try await getSourceKitD() + let path = scratchFilePath() + let positions = try await sourcekitd.openDocument( + path, + contents: """ + func test() { + 1️⃣} + struct MyStruct { + init(arg1: Int) {} + init(arg2: String) {} + } + """, + compilerArguments: [path] + ) + + // With 'addInitsToTopLevel' + let result1 = try await sourcekitd.completeOpen( + path: path, + position: positions["1️⃣"], + filter: "MyStr", + flags: [.addInitsToTopLevel] + ) + + XCTAssert(result1.items.filter({ $0.description.hasPrefix("MyStruct") }).count == 3) + let typeResult = try unwrap( + result1.items.first { $0.description == "MyStruct" && $0.kind == sourcekitd.values.declStruct } + ) + XCTAssertNotNil(typeResult.groupID) + XCTAssert( + result1.items.contains(where: { + $0.description.hasPrefix("MyStruct(arg1:") && $0.kind == sourcekitd.values.declConstructor + && $0.groupID == typeResult.groupID + }) + ) + XCTAssert( + result1.items.contains(where: { + $0.description.hasPrefix("MyStruct(arg2:") && $0.kind == sourcekitd.values.declConstructor + && $0.groupID == typeResult.groupID + }) + ) + XCTAssertLessThan( + try unwrap(result1.items.firstIndex(where: { $0.description == "MyStruct" })), + try unwrap(result1.items.firstIndex(where: { $0.description.hasPrefix("MyStruct(") })), + "Type names must precede the initializer calls" + ) + + // Without 'addInitsToTopLevel' + let result2 = try await sourcekitd.completeOpen( + path: path, + position: positions["1️⃣"], + filter: "MyStr", + flags: [] + ) + XCTAssert(result2.items.filter({ $0.description.hasPrefix("MyStruct") }).count == 1) + XCTAssertFalse(result2.items.contains(where: { $0.description.hasPrefix("MyStruct(") })) + } + + func testMembersGroupID() async throws { + let sourcekitd = try await getSourceKitD() + let path = scratchFilePath() + let positions = try await sourcekitd.openDocument( + path, + contents: """ + struct Animal { + var name: String + var species: String + func name(changedTo: String) { } + func name(updatedTo: String) { } + func otherFunction() { } + } + func test() { + let animal = Animal(name: "", species: "") + animal.1️⃣ + } + """, + compilerArguments: [path] + ) + + let result = try await sourcekitd.completeOpen( + path: path, + position: positions["1️⃣"], + filter: "" + ) + guard result.items.count == 6 else { + XCTFail("Expected 6 completion results; received \(result)") + return + } + + // Properties don't have a groupID. + XCTAssertEqual(result.items[0].name, "name") + XCTAssertNil(result.items[0].groupID) + XCTAssertEqual(result.items[1].name, "species") + XCTAssertNil(result.items[1].groupID) + + XCTAssertEqual(result.items[2].name, "name(changedTo:)") + XCTAssertEqual(result.items[3].name, "name(updatedTo:)") + XCTAssertEqual(result.items[2].groupID, result.items[3].groupID) + + XCTAssertEqual(result.items[4].name, "otherFunction()") + XCTAssertNotNil(result.items[4].groupID) + } + + func testAddCallWithNoDefaultArgs() async throws { + let sourcekitd = try await getSourceKitD() + let path = scratchFilePath() + let positions = try await sourcekitd.openDocument( + path, + contents: """ + struct Defaults { + func noDefault(a: Int) { } + func singleDefault(a: Int = 0) { } + } + func defaults(def: Defaults) { + def.1️⃣ + } + """, + compilerArguments: [path] + ) + + // With 'addCallWithNoDefaultArgs' + let result1 = try await sourcekitd.completeOpen( + path: path, + position: positions["1️⃣"], + filter: "", + flags: [.addCallWithNoDefaultArgs] + ) + guard result1.items.count == 4 else { + XCTFail("Expected 4 results; received \(result1)") + return + } + XCTAssertEqual(result1.items[0].description, "noDefault(a: Int)") + XCTAssertEqual(result1.items[1].description, "singleDefault()") + XCTAssertEqual(result1.items[2].description, "singleDefault(a: Int)") + XCTAssertEqual(result1.items[3].description, "self") + + // Without 'addCallWithNoDefaultArgs' + let result2 = try await sourcekitd.completeOpen( + path: path, + position: positions["1️⃣"], + filter: "" + ) + guard result2.items.count == 3 else { + XCTFail("Expected 3 results; received \(result2)") + return + } + XCTAssertEqual(result2.items[0].description, "noDefault(a: Int)") + XCTAssertEqual(result2.items[1].description, "singleDefault(a: Int)") + XCTAssertEqual(result2.items[2].description, "self") + } + + func testTextMatchScore() async throws { + let sourcekitd = try await getSourceKitD() + let path = scratchFilePath() + let positions = try await sourcekitd.openDocument( + path, + contents: """ + struct S { + func goodMatchOne() {} + func goodMatchNotOneButTwo() {} + func test() { + self.1️⃣ + } + } + """, + compilerArguments: [path] + ) + + let result1 = try await sourcekitd.completeOpen( + path: path, + position: positions["1️⃣"], + filter: "gmo" + ) + XCTAssertGreaterThan(result1.unfilteredResultCount, result1.items.count) + guard result1.items.count >= 2 else { + XCTFail("Expected at least 2 results; received \(result1)") + return + } + XCTAssertEqual(result1.items[0].description, "goodMatchOne()") + XCTAssertEqual(result1.items[1].description, "goodMatchNotOneButTwo()") + XCTAssertGreaterThan(result1.items[0].textMatchScore, result1.items[1].textMatchScore) + let result1Score = result1.items[0].textMatchScore + + let result2 = try await sourcekitd.completeOpen( + path: path, + position: positions["1️⃣"], + filter: "gmo", + useXPC: true + ) + guard result2.items.count >= 2 else { + XCTFail("Expected at least 2 results; received \(result2)") + return + } + XCTAssertEqual(result2.items[0].description, "goodMatchOne()") + XCTAssertEqual(result2.items[1].description, "goodMatchNotOneButTwo()") + XCTAssertGreaterThan(result2.items[0].textMatchScore, result2.items[1].textMatchScore) + XCTAssertEqual(result2.items[0].textMatchScore, result1Score) + } + + func testSemanticScore() async throws { + let sourcekitd = try await getSourceKitD() + let path = scratchFilePath() + let positions = try await sourcekitd.openDocument( + path, + contents: """ + struct S { + func goodMatchAsync() async {} + @available(*, deprecated) + func goodMatchDeprecated() {} + func goodMatchType() {} + func test() { + let goodMatchLocal = 1 + 1️⃣ + } + } + """, + compilerArguments: [path] + ) + + let result = try await sourcekitd.completeOpen( + path: path, + position: positions["1️⃣"], + filter: "goodMatch" + ) + guard result.items.count >= 4 else { + XCTFail("Expected at least 4 results; received \(result)") + return + } + XCTAssertEqual(result.items[0].description, "goodMatchLocal") + XCTAssertEqual(result.items[1].description, "goodMatchAsync() async") + XCTAssertEqual(result.items[2].description, "goodMatchType()") + XCTAssertEqual(result.items[3].description, "goodMatchDeprecated()") + XCTAssertGreaterThan(result.items[0].semanticScore, result.items[1].semanticScore) + // Note: async and deprecated get the same penalty currently, but we don't want to be too specific in this test. + XCTAssertEqual(result.items[1].semanticScore, result.items[2].semanticScore) + XCTAssertGreaterThan(result.items[1].semanticScore, result.items[3].semanticScore) + XCTAssertFalse(result.items[1].hasDiagnostic) + XCTAssertFalse(result.items[2].hasDiagnostic) + XCTAssertTrue(result.items[3].hasDiagnostic) + } + + func testSemanticScoreInit() async throws { + let sourcekitd = try await getSourceKitD() + let path = scratchFilePath() + let positions = try await sourcekitd.openDocument( + path, + contents: """ + enum E { + case good + init(_ param: Int, param2: String) { self = .good } + func test() { + let _: E = .1️⃣ + } + func test2() { + let local = 1 + E(2️⃣) + } + } + """, + compilerArguments: [path] + ) + + let result1 = try await sourcekitd.completeOpen( + path: path, + position: positions["1️⃣"], + filter: "" + ) + guard result1.items.count >= 2 else { + XCTFail("Expected at least 2 results; received \(result1)") + return + } + XCTAssertEqual(result1.items[0].description, "good") + XCTAssertEqual(result1.items[1].description, "init(param: Int, param2: String)") + XCTAssertGreaterThan(result1.items[0].semanticScore, result1.items[1].semanticScore) + + let result2 = try await sourcekitd.completeOpen( + path: path, + position: positions["2️⃣"], + filter: "" + ) + guard result2.items.count >= 2 else { + XCTFail("Expected at least 2 results; received \(result2)") + return + } + XCTAssertEqual(result2.items[0].description, "(param: Int, param2: String)") + XCTAssertEqual(result2.items[1].description, "local") + XCTAssertGreaterThan(result2.items[0].semanticScore, result2.items[1].semanticScore) + } + + func testSemanticScoreComponents() async throws { + let sourcekitd = try await getSourceKitD() + let path = scratchFilePath() + let positions = try await sourcekitd.openDocument( + path, + contents: """ + struct Animal { + var name = "Test" + func breed() { } + } + let animal = Animal() + animal.1️⃣ + """, + compilerArguments: [path] + ) + + let result = try await sourcekitd.completeOpen( + path: path, + position: positions["1️⃣"], + filter: "", + flags: [.includeSemanticComponents] + ) + + XCTAssertEqual(result.items.count, 3) + for item in result.items { + let data = Data(base64Encoded: try unwrap(item.semanticScoreComponents))! + let bytes = [UInt8](data) + XCTAssertFalse(bytes.isEmpty) + let classification = try SemanticClassification(byteRepresentation: bytes) + XCTAssertEqual(classification.score, item.semanticScore) + } + } + + func testMemberAccessTypes() async throws { + let sourcekitd = try await getSourceKitD() + let path = scratchFilePath(fileName: "AnimalKit.swift") + let positions = try await sourcekitd.openDocument( + path, + contents: """ + class Animal { } + class Dog: Animal { + var name = "Test" + func breed() { } + } + let dog = Dog() + dog1️⃣. + """, + compilerArguments: [path] + ) + + let result1 = try await sourcekitd.completeOpen( + path: path, + position: positions["1️⃣"], + filter: "" + ) + XCTAssertEqual(result1.memberAccessTypes, ["AnimalKit.Dog", "AnimalKit.Animal"]) + guard result1.items.count == 5 else { + XCTFail("Expected 5 result. Received \(result1)") + return + } + XCTAssertEqual(result1.items[0].module, "AnimalKit") + XCTAssertEqual(result1.items[0].name, "name") + XCTAssertEqual(result1.items[1].module, "AnimalKit") + XCTAssertEqual(result1.items[1].name, "breed()") + + let result2 = try await sourcekitd.completeUpdate( + path: path, + position: positions["1️⃣"], + filter: "name" + ) + XCTAssertEqual(result2.memberAccessTypes, ["AnimalKit.Dog", "AnimalKit.Animal"]) + guard result2.items.count == 1 else { + XCTFail("Expected 1 result. Received \(result2)") + return + } + XCTAssertEqual(result2.items[0].module, "AnimalKit") + XCTAssertEqual(result2.items[0].name, "name") + } + + func testTypeModule() async throws { + let sourcekitd = try await getSourceKitD() + let path = scratchFilePath(fileName: "AnimalKit.swift") + let positions = try await sourcekitd.openDocument( + path, + contents: """ + class Animal { } + class Dog: Animal { + var name = "Test" + func breed() { } + } + AnimalKit1️⃣. + """, + compilerArguments: [path] + ) + + let result = try await sourcekitd.completeOpen( + path: path, + position: positions["1️⃣"], + filter: "" + ) + XCTAssertEqual(result.memberAccessTypes, []) + XCTAssertEqual(result.items.count, 2) + // Note: the order of `Animal` and `Dog` isn't stable. + for item in result.items { + XCTAssertEqual(item.module, "AnimalKit") + } + } + + func testKeyword() async throws { + let sourcekitd = try await getSourceKitD() + let path = scratchFilePath() + let positions = try await sourcekitd.openDocument( + path, + contents: """ + 1️⃣ + """, + compilerArguments: [path] + ) + + let result = try await sourcekitd.completeOpen( + path: path, + position: positions["1️⃣"], + filter: "extensio" + ) + XCTAssertEqual(result.memberAccessTypes, []) + guard result.items.count == 1 else { + XCTFail("Expected 1 result. Received \(result)") + return + } + XCTAssertEqual(result.items[0].name, "extension") + XCTAssertNil(result.items[0].module) + } + + func testSemanticScoreComponentsAsExtraUpdate() async throws { + let sourcekitd = try await getSourceKitD() + let path = scratchFilePath() + let positions = try await sourcekitd.openDocument( + path, + contents: """ + struct S { + func test() { + self.1️⃣ + } + } + """, + compilerArguments: [path] + ) + + // Open without `includeSemanticComponents`. + let result1 = try await sourcekitd.completeOpen( + path: path, + position: positions["1️⃣"], + filter: "" + ) + XCTAssertEqual(result1.items.count, 2) + for item in result1.items { + XCTAssertNil(item.semanticScoreComponents) + } + + // Update without `includeSemanticComponents`. + let result2 = try await sourcekitd.completeUpdate( + path: path, + position: positions["1️⃣"], + filter: "t" + ) + XCTAssertEqual(result2.items.count, 1) + for item in result2.items { + XCTAssertNil(item.semanticScoreComponents) + } + + // Now, do the same update _with_ `includeSemanticComponents`. + let result3 = try await sourcekitd.completeUpdate( + path: path, + position: positions["1️⃣"], + filter: "t", + flags: [.includeSemanticComponents] + ) + XCTAssertEqual(result3.items.count, 1) + for item in result3.items { + // Assert we get `semanticScoreComponents`, + // when `update` is called with different options than `open`. + XCTAssertNotNil(item.semanticScoreComponents) + } + + // Same update _without_ `includeSemanticComponents`. + let result4 = try await sourcekitd.completeUpdate( + path: path, + position: positions["1️⃣"], + filter: "t" + ) + XCTAssertEqual(result4.items.count, 1) + for item in result4.items { + // Response no longer contains the `semanticScoreComponents`. + XCTAssertNil(item.semanticScoreComponents) + } + } + + // rdar://104381080 (NSImage(imageLiteralResourceName:) was my top completion — this seems odd) + func testPopularityForTypeFromSubmodule() async throws { + let sourcekitd = try await getSourceKitD() + let path = scratchFilePath() + let positions = try await sourcekitd.openDocument( + path, + contents: """ + import AppKit + func test() { + 1️⃣ + } + """, + compilerArguments: [path] + ) + + let popularityIndex = """ + { + "AppKit": { + "scores": [ + 0.6 + ], + "values": [ + "NSImage" + ] + } + } + """ + try await withTestScratchDir { scratchDir in + let popularityIndexPath = scratchDir.appending(component: "popularityIndex.json") + try popularityIndex.write(to: popularityIndexPath, atomically: true, encoding: .utf8) + try await sourcekitd.setPopularityIndex( + scopedPopularityDataPath: try popularityIndexPath.filePath, + popularModules: [], + notoriousModules: [] + ) + + let result = try await sourcekitd.completeOpen( + path: path, + position: positions["1️⃣"], + filter: "", + flags: [.addInitsToTopLevel] + ) + // `NSImage` is defined in `AppKit.NSImage` (a submodule). + // The popularity index is keyed only on the base module (e.g. `AppKit`). + // This asserts we correctly see `NSImage` as popular. + XCTAssertEqual(result.items.first?.description, "NSImage") + } + } + + func testPopularity() async throws { + let sourcekitd = try await getSourceKitD() + let path = scratchFilePath() + let positions = try await sourcekitd.openDocument( + path, + contents: """ + struct S { + func test() { + self.1️⃣ + } + let popular: Int = 0 + let other1: Int = 0 + let unpopular: Int = 0 + let other2: Int = 0 + let recent1: Int = 0 + let recent2: Int = 0 + let other3: Int = 0 + } + """, + compilerArguments: [path] + ) + + // Reset the scoped popularity data path if it was set by previous requests + try await sourcekitd.setPopularityIndex( + scopedPopularityDataPath: "/invalid", + popularModules: [], + notoriousModules: [] + ) + + try await sourcekitd.setPopularAPI(popular: ["popular"], unpopular: ["unpopular"]) + + let result1 = try await sourcekitd.completeOpen( + path: path, + position: positions["1️⃣"], + filter: "", + recentCompletions: ["recent1"] + ) + guard result1.items.count >= 6 else { + XCTFail("Expected at least 6 results. Received \(result1)") + return + } + XCTAssertEqual(result1.items[0].description, "popular") + XCTAssertEqual(result1.items[1].description, "recent1") + XCTAssertEqual(result1.items[2].description, "other1") + XCTAssertEqual(result1.items.last?.description, "unpopular") + + let result2 = try await sourcekitd.completeOpen( + path: path, + position: positions["1️⃣"], + filter: "", + recentCompletions: ["recent2", "recent1"] + ) + + guard result2.items.count >= 6 else { + XCTFail("Expected at least 6 results. Received \(result2)") + return + } + XCTAssertEqual(result2.items[0].description, "popular") + XCTAssertEqual(result2.items[1].description, "recent2") + XCTAssertEqual(result2.items[2].description, "recent1") + XCTAssertEqual(result2.items[3].description, "other1") + XCTAssertEqual(result2.items.last?.description, "unpopular") + + try await sourcekitd.setPopularAPI(popular: [], unpopular: []) + + let result3 = try await sourcekitd.completeOpen( + path: path, + position: positions["1️⃣"], + filter: "", + recentCompletions: ["recent2", "recent1"] + ) + guard result3.items.count >= 6 else { + XCTFail("Expected at least 6 results. Received \(result3)") + return + } + XCTAssertEqual(result3.items[0].description, "recent2") + XCTAssertEqual(result3.items[1].description, "recent1") + // Results 2 - 6 share the same score + XCTAssertEqual(result3.items[2].description, "popular") + XCTAssertEqual(result3.items[3].description, "other1") + XCTAssertEqual(result3.items[4].description, "unpopular") + XCTAssertEqual(result3.items[5].description, "other2") + XCTAssertEqual(result3.items[6].description, "other3") + XCTAssertEqual(result3.items[2].semanticScore, result3.items[3].semanticScore) + XCTAssertEqual(result3.items[2].semanticScore, result3.items[4].semanticScore) + XCTAssertEqual(result3.items[2].semanticScore, result3.items[5].semanticScore) + XCTAssertEqual(result3.items[2].semanticScore, result3.items[6].semanticScore) + } + + func testScopedPopularity() async throws { + let sourcekitd = try await getSourceKitD() + let path = scratchFilePath() + let positions = try await sourcekitd.openDocument( + path, + contents: """ + struct Strinz: Encodable { + var aProp: String = "" + var value: Int = 12 + } + func test(arg: Strinz) { + arg.1️⃣ + } + func testGlobal() { + 2️⃣ + } + """, + compilerArguments: [path, "-module-name", "MyMod"] + ) + + let popularityIndex = """ + { + "Swift.Encodable": { + "values": [ + "encode" + ], + "scores": [ + 1.0 + ] + } + } + """ + + try await withTestScratchDir { scratchDir in + let popularityIndexPath = scratchDir.appending(component: "popularityIndex.json") + try popularityIndex.write(to: popularityIndexPath, atomically: true, encoding: .utf8) + + let result1 = try await sourcekitd.completeOpen( + path: path, + position: positions["1️⃣"], + filter: "" + ) + let scoreWithoutPopularity = try XCTUnwrap(result1.items.first(where: { $0.name == "encode(to:)" })).semanticScore + + try await sourcekitd.setPopularityIndex( + scopedPopularityDataPath: try popularityIndexPath.filePath, + popularModules: [], + notoriousModules: ["MyMod"] + ) + + let result2 = try await sourcekitd.completeOpen( + path: path, + position: positions["1️⃣"], + filter: "" + ) + let scoreWithPopularity = try XCTUnwrap(result2.items.first(where: { $0.name == "encode(to:)" })).semanticScore + + XCTAssert(scoreWithoutPopularity < scoreWithPopularity) + + // Ensure 'notoriousModules' lowers the score. + let result3 = try await sourcekitd.completeOpen( + path: path, + position: positions["2️⃣"], + filter: "Strin", + recentCompletions: [] + ) + let string = try XCTUnwrap(result3.items.first(where: { $0.name == "String" })) + let strinz = try XCTUnwrap(result3.items.first(where: { $0.name == "Strinz" })) + XCTAssert(string.textMatchScore == strinz.textMatchScore) + XCTAssert(string.semanticScore > strinz.semanticScore) + } + } + + func testModulePopularity() async throws { + let sourcekitd = try await getSourceKitD() + let path = scratchFilePath() + let positions = try await sourcekitd.openDocument( + path, + contents: """ + struct S { + func test() { + self.1️⃣ + } + let foo: Int = 0 + } + """, + compilerArguments: [path] + ) + + try await sourcekitd.setPopularityTable( + PopularityTable(moduleSymbolReferenceTables: [], recentCompletions: [], popularModules: [], notoriousModules: []) + ) + + let result1 = try await sourcekitd.completeOpen( + path: path, + position: positions["1️⃣"], + filter: "", + recentCompletions: [] + ) + let noPopularModulesScore = try unwrap(result1.items.first(where: { $0.description == "foo" })?.semanticScore) + + try await sourcekitd.setPopularityTable( + PopularityTable( + moduleSymbolReferenceTables: [], + recentCompletions: [], + popularModules: ["a"], + notoriousModules: [] + ) + ) + + let result2 = try await sourcekitd.completeOpen( + path: path, + position: positions["1️⃣"], + filter: "", + recentCompletions: [] + ) + let moduleIsPopularScore = try unwrap(result2.items.first(where: { $0.description == "foo" })?.semanticScore) + + try await sourcekitd.setPopularityTable( + PopularityTable( + moduleSymbolReferenceTables: [], + recentCompletions: [], + popularModules: [], + notoriousModules: ["a"] + ) + ) + + let result3 = try await sourcekitd.completeOpen( + path: path, + position: positions["1️⃣"], + filter: "", + recentCompletions: [] + ) + let moduleIsUnpopularScore = try unwrap(result3.items.first(where: { $0.description == "foo" })?.semanticScore) + + XCTAssertLessThan(moduleIsUnpopularScore, noPopularModulesScore) + XCTAssertLessThan(noPopularModulesScore, moduleIsPopularScore) + } + + func testFlair() async throws { + let sourcekitd = try await getSourceKitD() + let path = scratchFilePath() + let positions = try await sourcekitd.openDocument( + path, + contents: """ + struct S { + func foo(x: Int, y: Int) {} + func foo(_ arg: String) {} + func test(localArg: String) { + self.foo(1️⃣) + } + } + """, + compilerArguments: [path] + ) + + let result = try await sourcekitd.completeOpen( + path: path, + position: positions["1️⃣"], + filter: "" + ) + guard result.items.count >= 3 else { + XCTFail("Expected at least 3 results. Received \(result)") + return + } + XCTAssertTrue(Set(result.items[0...1].map(\.description)) == ["(arg: String)", "(x: Int, y: Int)"]) + XCTAssertTrue(result.items[2...].contains(where: { $0.description == "localArg" })) + } + + func testPluginFilterAndSortPerfAllMatch() async throws { + let sourcekitd = try await getSourceKitD() + let path = scratchFilePath() + let (position, recent) = try await sourcekitd.perfTestSetup(path: path) + + let initResult = try await sourcekitd.completeOpen( + path: path, + position: position, + filter: "", + recentCompletions: recent + ) + XCTAssertEqual(initResult.items.count, 200) + + self.measureMetrics([.wallClockTime], automaticallyStartMeasuring: false) { + assertNoThrow { + self.startMeasuring() + let result = try runAsync { + return try await sourcekitd.completeUpdate( + path: path, + position: position, + filter: "" + ) + } + self.stopMeasuring() + XCTAssertEqual(result.items.count, 200) + + try runAsync { + // Use a non-matching search to ensure we aren't caching the results. + let resetResult = try await sourcekitd.completeUpdate( + path: path, + position: position, + filter: "sadfasdfasd" + ) + XCTAssertEqual(resetResult.items.count, 0) + } + } + } + } + + func testPluginFilterAndSortPerfFiltered() async throws { + let sourcekitd = try await getSourceKitD() + let path = scratchFilePath() + let (position, recent) = try await sourcekitd.perfTestSetup(path: path) + + let initResult = try await sourcekitd.completeOpen( + path: path, + position: position, + filter: "", + recentCompletions: recent + ) + XCTAssertGreaterThanOrEqual(initResult.unfilteredResultCount, initResult.items.count) + + self.measureMetrics([.wallClockTime], automaticallyStartMeasuring: false) { + assertNoThrow { + self.startMeasuring() + let result = try runAsync { + try await sourcekitd.completeUpdate( + path: path, + position: position, + filter: "mMethS" + ) + } + self.stopMeasuring() + XCTAssertEqual(result.items.count, 200) + + try runAsync { + // Use a non-matching search to ensure we aren't caching the results. + let resetResult = try await sourcekitd.completeUpdate( + path: path, + position: position, + filter: "sadfasdfasd" + ) + XCTAssertEqual(resetResult.items.count, 0) + } + } + } + } + + func testCrossModuleCompletion() async throws { + let project = try await PluginSwiftPMTestProject(files: [ + "Sources/LibA/LibA.swift": """ + public struct LibA { + public init() {} + public func method() {} + } + """, + "Sources/LibB/LibB.swift": """ + import LibA + func test(lib: LibA) { + lib.1️⃣method() + } + """, + "Package.swift": """ + // swift-tools-version: 5.7 + + import PackageDescription + + let package = Package( + name: "MyLibrary", + targets: [ + .target(name: "LibA"), + .target(name: "LibB", dependencies: ["LibA"]), + ] + ) + """, + ]) + + // Open document in sourcekitd + let sourcekitd = try await getSourceKitD() + let libBPath = try project.uri(for: "LibB.swift").pseudoPath + try await sourcekitd.openDocument( + libBPath, + contents: project.contents(of: "LibB.swift"), + compilerArguments: project.compilerArguments(for: "LibB.swift") + ) + + // Invoke first code completion + let result = try await sourcekitd.completeOpen( + path: libBPath, + position: project.position(of: "1️⃣", in: "LibB.swift"), + filter: "met" + ) + XCTAssertEqual(1, result.items.count) + XCTAssertEqual(result.items.first?.name, "method()") + + // Modify LibA.swift to contain another memeber on the `LibA` struct + let modifiedLibA = """ + \(try project.contents(of: "LibA.swift")) + extension LibA { + public var meta: Int { 0 } + } + """ + try modifiedLibA.write(to: project.uri(for: "LibA.swift").fileURL!, atomically: true, encoding: .utf8) + try await SwiftPMTestProject.build(at: project.scratchDirectory) + + // Tell sourcekitd that dependencies have been updated and run completion again. + try await sourcekitd.dependencyUpdated() + let result2 = try await sourcekitd.completeOpen( + path: libBPath, + position: project.position(of: "1️⃣", in: "LibB.swift"), + filter: "met" + ) + XCTAssertEqual(Set(result2.items.map(\.name)), ["meta", "method()"]) + } + + func testCompletionImportDepth() async throws { + let project = try await PluginSwiftPMTestProject(files: [ + "Sources/Main/Main.swift": """ + import Depth1Module + + struct Depth0Struct {} + + func test() { + 1️⃣ + return + } + """, + "Sources/Depth1Module/Depth1.swift": """ + @_exported import Depth2Module + public struct Depth1Struct {} + """, + "Sources/Depth2Module/Depth2.swift": """ + public struct Depth2Struct {} + """, + "Package.swift": """ + // swift-tools-version: 5.7 + + import PackageDescription + + let package = Package( + name: "MyLibrary", + targets: [ + .executableTarget(name: "Main", dependencies: ["Depth1Module"]), + .target(name: "Depth1Module", dependencies: ["Depth2Module"]), + .target(name: "Depth2Module"), + ] + ) + """, + ]) + + let sourcekitd = try await getSourceKitD() + let mainPath = try project.uri(for: "Main.swift").pseudoPath + try await sourcekitd.openDocument( + mainPath, + contents: project.contents(of: "Main.swift"), + compilerArguments: project.compilerArguments(for: "Main.swift") + ) + + let result = try await sourcekitd.completeOpen( + path: mainPath, + position: project.position(of: "1️⃣", in: "Main.swift"), + filter: "depth" + ) + + let depth0struct = try unwrap(result.items.first(where: { $0.name == "Depth0Struct" })) + let depth1struct = try unwrap(result.items.first(where: { $0.name == "Depth1Struct" })) + let depth2struct = try unwrap(result.items.first(where: { $0.name == "Depth2Struct" })) + let depth1module = try unwrap(result.items.first(where: { $0.name == "Depth1Module" })) + let depth2module = try unwrap(result.items.first(where: { $0.name == "Depth2Module" })) + + XCTAssertGreaterThan(depth0struct.semanticScore, depth1struct.semanticScore) + XCTAssertGreaterThan(depth1struct.semanticScore, depth2struct.semanticScore) + + // Since "module" entry doesn't have "import depth", we only checks that modules are de-prioritized. + XCTAssertGreaterThan(depth2struct.semanticScore, depth1module.semanticScore) + XCTAssertGreaterThan(depth2struct.semanticScore, depth2module.semanticScore) + } + + func testCompletionDiagnostics() async throws { + let sourcekitd = try await getSourceKitD() + let path = scratchFilePath() + let positions = try await sourcekitd.openDocument( + path, + contents: """ + import 1️⃣Swift + struct S: P { + @available(*, deprecated) + func deprecatedF() {} + func test() { + self.2️⃣ + } + var theVariable: Int { + 3️⃣ + } + @available(macOS, deprecated: 100000.0) + func softDeprecatedF() {} + } + """, + compilerArguments: [path] + ) + + let result1 = try await sourcekitd.completeOpen(path: path, position: positions["1️⃣"], filter: "Swift") + let swiftResult = try unwrap(result1.items.filter({ $0.description == "Swift" }).first) + XCTAssertEqual(swiftResult.description, "Swift") + XCTAssertEqual(swiftResult.hasDiagnostic, true) + + let diag1 = try unwrap(try await sourcekitd.completeDiagnostic(id: swiftResult.id)) + + XCTAssertEqual(diag1.severity, sourcekitd.values.diagWarning) + XCTAssertEqual(diag1.description, "module 'Swift' is already imported") + + let result2 = try await sourcekitd.completeOpen(path: path, position: positions["2️⃣"], filter: "deprecatedF") + guard result2.items.count >= 2 else { + XCTFail("Expected at least 2 results. Received \(result2)") + return + } + + XCTAssertEqual(result2.items[0].description, "deprecatedF()") + XCTAssertEqual(result2.items[0].hasDiagnostic, true) + let diag2_0 = try unwrap(try await sourcekitd.completeDiagnostic(id: result2.items[0].id)) + XCTAssertEqual(diag2_0.severity, sourcekitd.values.diagWarning) + XCTAssertEqual(diag2_0.description, "'deprecatedF()' is deprecated") + + XCTAssertEqual(result2.items[1].description, "softDeprecatedF()") + XCTAssertEqual(result2.items[1].hasDiagnostic, true) + let diag2_1 = try unwrap(try await sourcekitd.completeDiagnostic(id: result2.items[1].id)) + XCTAssertEqual(diag2_1.severity, sourcekitd.values.diagWarning) + XCTAssertEqual(diag2_1.description, "'softDeprecatedF()' will be deprecated in a future version of macOS") + + let result4 = try await sourcekitd.completeOpen(path: path, position: positions["3️⃣"], filter: "theVariable") + guard result4.items.count >= 1 else { + XCTFail("Expected at least 1 results. Received \(result4)") + return + } + XCTAssertEqual(result4.items[0].description, "theVariable") + XCTAssertEqual(result4.items[0].hasDiagnostic, true) + + let diag4_0 = try unwrap(try await sourcekitd.completeDiagnostic(id: result4.items[0].id)) + + XCTAssertEqual(diag4_0.severity, sourcekitd.values.diagWarning) + XCTAssertEqual(diag4_0.description, "attempting to access 'theVariable' within its own getter") + } + + func testActorKind() async throws { + let sourcekitd = try await getSourceKitD() + let path = scratchFilePath() + let positions = try await sourcekitd.openDocument( + path, + contents: """ + actor MyActor { + } + func test() { + 1️⃣} + """, + compilerArguments: [path] + ) + + let result = try await sourcekitd.completeOpen(path: path, position: positions["1️⃣"], filter: "My") + let actorItem = try unwrap(result.items.first { item in item.description == "MyActor" }) + XCTAssertEqual(actorItem.kind, sourcekitd.values.declActor) + } + + func testMacroKind() async throws { + let sourcekitd = try await getSourceKitD() + let path = scratchFilePath() + let positions = try await sourcekitd.openDocument( + path, + contents: """ + @attached(conformance) public macro MyConformance() = #externalMacro(module: "MyMacros", type: "MyConformanceMacro") + @freestanding(expression) public macro MyExpression(_: Any) -> String = #externalMacro(module: "MyMacros", type: "MyExpressionMacro") + + func testAttached() { + @1️⃣ + } + func testFreestanding() { + _ = #2️⃣ + } + """, + compilerArguments: [path] + ) + + let result1 = try await sourcekitd.completeOpen(path: path, position: positions["1️⃣"], filter: "My") + let macroItem1 = try unwrap(result1.items.first { item in item.name == "MyConformance" }) + XCTAssertEqual(macroItem1.kind, sourcekitd.values.declMacro) + + let result2 = try await sourcekitd.completeOpen(path: path, position: positions["2️⃣"], filter: "My") + let macroItem2 = try unwrap(result2.items.first { item in item.name.starts(with: "MyExpression") }) + XCTAssertEqual(macroItem2.kind, sourcekitd.values.declMacro) + + } + + func testMaxResults() async throws { + let sourcekitd = try await getSourceKitD() + let path = scratchFilePath() + var sourceText = "//dummy\n"; + for i in 0..<200 { + /// Create at least 200 (default maxResults) items + sourceText += """ + func foo\(i)() {} + + """ + } + try await sourcekitd.openDocument(path, contents: sourceText, compilerArguments: [path]) + + let position = Position(line: 200, utf16index: 0) + + let result1 = try await sourcekitd.completeOpen(path: path, position: position, filter: "f", maxResults: 3) + XCTAssertEqual(result1.items.count, 3) + + let result2 = try await sourcekitd.completeUpdate(path: path, position: position, filter: "fo", maxResults: 5) + XCTAssertEqual(result2.items.count, 5) + + let result3 = try await sourcekitd.completeUpdate(path: path, position: position, filter: "f") + XCTAssertEqual(result3.items.count, 200) + } +} + +// MARK: - Structured result types + +fileprivate struct CompletionResultSet: Sendable { + var unfilteredResultCount: Int + var memberAccessTypes: [String] + var items: [CompletionResult] + + init(_ dict: SKDResponseDictionary) throws { + let keys = dict.sourcekitd.keys + guard let unfilteredResultCount: Int = dict[keys.unfilteredResultCount], + + let memberAccessTypes = dict[keys.memberAccessTypes]?.asStringArray, + let results: SKDResponseArray = dict[keys.results] + else { + throw TestError( + "expected {key.results: , key.unfiltered_result_count: }; got \(dict)" + ) + } + + self.unfilteredResultCount = unfilteredResultCount + self.memberAccessTypes = memberAccessTypes + self.items = + try results + .map { try CompletionResult($0) } + .sorted(by: { $0.semanticScore > $1.semanticScore }) + + XCTAssertGreaterThanOrEqual( + self.unfilteredResultCount, + self.items.count, + "unfiltered_result_count must be greater than or equal to the count of results" + ) + } +} + +fileprivate struct CompletionResult: Equatable, Sendable { + nonisolated(unsafe) var kind: sourcekitd_api_uid_t + var id: Int + var name: String + var description: String + var sourcetext: String + var module: String? + var typename: String + var textMatchScore: Double + var semanticScore: Double + var semanticScoreComponents: String? + var priorityBucket: Int + var isSystem: Bool + var numBytesToErase: Int + var hasDiagnostic: Bool + var groupID: Int? + + init(_ dict: SKDResponseDictionary) throws { + let keys = dict.sourcekitd.keys + + guard let kind: sourcekitd_api_uid_t = dict[keys.kind], + let id: Int = dict[keys.identifier], + let name: String = dict[keys.name], + let description: String = dict[keys.description], + let sourcetext: String = dict[keys.sourceText], + let typename: String = dict[keys.typeName], + let textMatchScore: Double = dict[keys.textMatchScore], + let semanticScore: Double = dict[keys.semanticScore], + let priorityBucket: Int = dict[keys.priorityBucket], + let isSystem: Bool = dict[keys.isSystem], + let hasDiagnostic: Bool = dict[keys.hasDiagnostic] + else { + throw TestError("Failed to decode CompletionResult. Received \(dict)") + } + + self.kind = kind + self.id = id + self.name = name + self.description = description + self.sourcetext = sourcetext + self.module = dict[keys.moduleName] + self.typename = typename + self.textMatchScore = textMatchScore + self.semanticScore = semanticScore + self.semanticScoreComponents = dict[keys.semanticScoreComponents] + self.priorityBucket = priorityBucket + self.isSystem = isSystem + self.numBytesToErase = dict[keys.numBytesToErase] ?? 0 + self.hasDiagnostic = hasDiagnostic + self.groupID = dict[keys.groupId] + assert(self.groupID != 0) + } +} + +fileprivate struct CompletionDocumentation { + var docBrief: String? = nil + var associatedUSRs: [String] = [] + + init(_ dict: SKDResponseDictionary) { + let keys = dict.sourcekitd.keys + self.docBrief = dict[keys.docBrief] + self.associatedUSRs = dict[keys.associatedUSRs]?.asStringArray ?? [] + } +} + +fileprivate struct CompletionDiagnostic { + var severity: sourcekitd_api_uid_t + var description: String + + init?(_ dict: SKDResponseDictionary) { + let keys = dict.sourcekitd.keys + guard + let severity: sourcekitd_api_uid_t = dict[keys.severity], + let description: String = dict[keys.description] + else { + return nil + } + self.severity = severity + self.description = description + } +} + +fileprivate struct TestError: Error { + let error: String + + init(_ message: String) { + self.error = message + } +} + +// MARK: - sourcekitd convenience functions + +struct CompletionRequestFlags: OptionSet { + let rawValue: Int + static let annotate: Self = .init(rawValue: 1 << 0) + static let addInitsToTopLevel: Self = .init(rawValue: 1 << 1) + static let addCallWithNoDefaultArgs: Self = .init(rawValue: 1 << 2) + static let includeSemanticComponents: Self = .init(rawValue: 1 << 3) +} + +fileprivate extension SourceKitD { + @discardableResult + func openDocument( + _ name: String, + contents markedSource: String, + compilerArguments: [String]? = nil + ) async throws -> DocumentPositions { + let (markers, textWithoutMarkers) = extractMarkers(markedSource) + var compilerArguments = compilerArguments ?? [name] + if let defaultSDKPath { + compilerArguments += ["-sdk", defaultSDKPath] + } + let req = dictionary([ + keys.request: requests.editorOpen, + keys.name: name, + keys.sourceText: textWithoutMarkers, + keys.syntacticOnly: 1, + keys.compilerArgs: compilerArguments as [SKDRequestValue], + ]) + _ = try await send(req, timeout: .seconds(defaultTimeout), fileContents: nil) + return DocumentPositions(markers: markers, textWithoutMarkers: textWithoutMarkers) + } + + func editDocument(_ name: String, fromOffset offset: Int, length: Int, newContents: String) async throws { + let req = dictionary([ + keys.request: requests.editorReplaceText, + keys.name: name, + keys.offset: offset, + keys.length: length, + keys.sourceText: newContents, + keys.syntacticOnly: 1, + ]) + + _ = try await send(req, timeout: .seconds(defaultTimeout), fileContents: nil) + } + + func closeDocument(_ name: String) async throws { + let req = dictionary([ + keys.request: requests.editorClose, + keys.name: name, + ]) + + _ = try await send(req, timeout: .seconds(defaultTimeout), fileContents: nil) + } + + func completeImpl( + requestUID: sourcekitd_api_uid_t, + path: String, + position: Position, + filter: String, + recentCompletions: [String]? = nil, + flags: CompletionRequestFlags = [], + useXPC: Bool = false, + maxResults: Int? = nil, + compilerArguments: [String]? = nil + ) async throws -> CompletionResultSet { + let options = dictionary([ + keys.useNewAPI: 1, + keys.annotatedDescription: flags.contains(.annotate) ? 1 : 0, + keys.addInitsToTopLevel: flags.contains(.addInitsToTopLevel) ? 1 : 0, + keys.addCallWithNoDefaultArgs: flags.contains(.addCallWithNoDefaultArgs) ? 1 : 0, + keys.includeSemanticComponents: flags.contains(.includeSemanticComponents) ? 1 : 0, + keys.filterText: filter, + keys.recentCompletions: recentCompletions as [SKDRequestValue]?, + keys.maxResults: maxResults, + ]) + + let req = dictionary([ + keys.request: requestUID, + keys.line: position.line + 1, + // Technically sourcekitd needs a UTF-8 index but we can assume there are no Unicode characters in the tests + keys.column: position.utf16index + 1, + keys.sourceFile: path, + keys.codeCompleteOptions: options, + keys.compilerArgs: compilerArguments as [SKDRequestValue]?, + ]) + + let res = try await send(req, timeout: .seconds(defaultTimeout), fileContents: nil) + return try CompletionResultSet(res) + } + + func completeOpen( + path: String, + position: Position, + filter: String, + recentCompletions: [String]? = nil, + flags: CompletionRequestFlags = [], + useXPC: Bool = false, + maxResults: Int? = nil, + compilerArguments: [String]? = nil + ) async throws -> CompletionResultSet { + return try await completeImpl( + requestUID: requests.codeCompleteOpen, + path: path, + position: position, + filter: filter, + recentCompletions: recentCompletions, + flags: flags, + useXPC: useXPC, + maxResults: maxResults, + compilerArguments: compilerArguments + ) + } + + func completeUpdate( + path: String, + position: Position, + filter: String, + flags: CompletionRequestFlags = [], + useXPC: Bool = false, + maxResults: Int? = nil + ) async throws -> CompletionResultSet { + return try await completeImpl( + requestUID: requests.codeCompleteUpdate, + path: path, + position: position, + filter: filter, + recentCompletions: nil, + flags: flags, + useXPC: useXPC, + maxResults: maxResults, + compilerArguments: nil + ) + } + + func completeClose(path: String, position: Position) async throws { + let req = dictionary([ + keys.request: requests.codeCompleteClose, + keys.line: position.line + 1, + // Technically sourcekitd needs a UTF-8 index but we can assume there are no Unicode characters in the tests + keys.column: position.utf16index + 1, + keys.sourceFile: path, + keys.codeCompleteOptions: dictionary([keys.useNewAPI: 1]), + ]) + + _ = try await send(req, timeout: .seconds(defaultTimeout), fileContents: nil) + } + + func completeDocumentation(id: Int) async throws -> CompletionDocumentation { + let req = dictionary([ + keys.request: requests.codeCompleteDocumentation, + keys.identifier: id, + ]) + + let resp = try await send(req, timeout: .seconds(defaultTimeout), fileContents: nil) + return CompletionDocumentation(resp) + } + + func completeDiagnostic(id: Int) async throws -> CompletionDiagnostic? { + let req = dictionary([ + keys.request: requests.codeCompleteDiagnostic, + keys.identifier: id, + ]) + let resp = try await send(req, timeout: .seconds(defaultTimeout), fileContents: nil) + + return CompletionDiagnostic(resp) + } + + func dependencyUpdated() async throws { + let req = dictionary([ + keys.request: requests.dependencyUpdated + ]) + _ = try await send(req, timeout: .seconds(defaultTimeout), fileContents: nil) + } + + func setPopularAPI(popular: [String], unpopular: [String]) async throws { + let req = dictionary([ + keys.request: requests.codeCompleteSetPopularAPI, + keys.codeCompleteOptions: dictionary([keys.useNewAPI: 1]), + keys.popular: popular as [SKDRequestValue], + keys.unpopular: unpopular as [SKDRequestValue], + ]) + + let resp = try await send(req, timeout: .seconds(defaultTimeout), fileContents: nil) + XCTAssertEqual(resp[keys.useNewAPI], 1) + } + + func setPopularityIndex( + scopedPopularityDataPath: String, + popularModules: [String], + notoriousModules: [String] + ) async throws { + let req = dictionary([ + keys.request: requests.codeCompleteSetPopularAPI, + keys.codeCompleteOptions: dictionary([keys.useNewAPI: 1]), + keys.scopedPopularityTablePath: scopedPopularityDataPath, + keys.popularModules: popularModules as [SKDRequestValue], + keys.notoriousModules: notoriousModules as [SKDRequestValue], + ]) + + let resp = try await send(req, timeout: .seconds(defaultTimeout), fileContents: nil) + XCTAssertEqual(resp[keys.useNewAPI], 1) + } + + func setPopularityTable(_ popularityTable: PopularityTable) async throws { + let symbolPopularity = popularityTable.symbolPopularity.map { key, value in + dictionary([ + keys.popularityKey: key, + keys.popularityValueIntBillion: Int(value.scoreComponent * 1_000_000_000), + ]) + } + let modulePopularity = popularityTable.modulePopularity.map { key, value in + dictionary([ + keys.popularityKey: key, + keys.popularityValueIntBillion: Int(value.scoreComponent * 1_000_000_000), + ]) + } + let req = dictionary([ + keys.request: requests.codeCompleteSetPopularAPI, + keys.codeCompleteOptions: dictionary([keys.useNewAPI: 1]), + keys.symbolPopularity: symbolPopularity as [SKDRequestValue], + keys.modulePopularity: modulePopularity as [SKDRequestValue], + ]) + + let resp = try await send(req, timeout: .seconds(defaultTimeout), fileContents: nil) + XCTAssertEqual(resp[keys.useNewAPI], 1) + } + + func perfTestSetup(path: String) async throws -> (Position, recent: [String]) { + var content = """ + struct S { + func test() { + self.1️⃣ + } + """ + + #if DEBUG + let numMethods = 1_000 + #else + let numMethods = 100_000 + #endif + + var popular: [String] = [] + var unpopular: [String] = [] + + for i in 0..(_ body: @escaping @Sendable () async throws -> T) throws -> T { + var result: Result! + let expectation = XCTestExpectation(description: "") + Task { + do { + result = .success(try await body()) + } catch { + result = .failure(error) + } + expectation.fulfill() + } + let started = XCTWaiter.wait(for: [expectation], timeout: defaultTimeout) + if started != .completed { + throw ExpectationNotFulfilledError() + } + return try result.get() +} diff --git a/Utilities/build-script-helper.py b/Utilities/build-script-helper.py index 0440080b4..6c7ea7e5e 100755 --- a/Utilities/build-script-helper.py +++ b/Utilities/build-script-helper.py @@ -15,7 +15,7 @@ # General utilities -def fatal_error(message): +def fatal_error(message: str): print(message, file=sys.stderr) raise SystemExit(1) @@ -85,7 +85,7 @@ def get_build_target(swift_exec: str, args: argparse.Namespace, cross_compile: b def get_swiftpm_options(swift_exec: str, args: argparse.Namespace, suppress_verbose: bool = False) -> List[str]: - swiftpm_args = [ + swiftpm_args: List[str] = [ '--package-path', args.package_path, '--scratch-path', args.build_path, '--configuration', args.configuration, @@ -153,7 +153,7 @@ def get_swiftpm_environment_variables(swift_exec: str, args: argparse.Namespace) 'swift test' invocation. """ - env = { + env: Dict[str, str] = { # Set the toolchain used in tests at runtime 'SOURCEKIT_TOOLCHAIN_PATH': args.toolchain, 'INDEXSTOREDB_TOOLCHAIN_BIN_PATH': args.toolchain, @@ -241,14 +241,18 @@ def install_binary(exe: str, source_dir: str, install_dir: str, verbose: bool) - def install(swift_exec: str, args: argparse.Namespace) -> None: - build_single_product('sourcekit-lsp', swift_exec, args) - swiftpm_args = get_swiftpm_options(swift_exec, args) additional_env = get_swiftpm_environment_variables(swift_exec, args) bin_path = swiftpm_bin_path(swift_exec, swiftpm_args=swiftpm_args, additional_env=additional_env) + build_single_product('sourcekit-lsp', swift_exec, args) + build_single_product('SwiftSourceKitPlugin', swift_exec, args) + build_single_product('SwiftSourceKitClientPlugin', swift_exec, args) + for prefix in args.install_prefixes: install_binary('sourcekit-lsp', bin_path, os.path.join(prefix, 'bin'), verbose=args.verbose) + install_binary('libSwiftSourceKitPlugin.dylib', bin_path, os.path.join(prefix, 'bin'), verbose=args.verbose) + install_binary('libSwiftSourceKitClientPlugin.dylib', bin_path, os.path.join(prefix, 'bin'), verbose=args.verbose) def handle_invocation(swift_exec: str, args: argparse.Namespace) -> None: From 69dd6558c01b24f3b4d2b5e02b7dcf416b85e47b Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Tue, 7 Jan 2025 08:07:17 -0800 Subject: [PATCH 02/18] Simplify rangeOf --- .../Utilities/SwiftExtensions.swift | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/Sources/CompletionScoring/Utilities/SwiftExtensions.swift b/Sources/CompletionScoring/Utilities/SwiftExtensions.swift index 7a624cfaa..a8e95c118 100644 --- a/Sources/CompletionScoring/Utilities/SwiftExtensions.swift +++ b/Sources/CompletionScoring/Utilities/SwiftExtensions.swift @@ -429,13 +429,17 @@ extension UnsafeBufferPointer { extension UnsafeBufferPointer { package func rangeOf(bytes needle: UnsafeBufferPointer, startOffset: Int = 0) -> Range? { - baseAddress.flatMap { baseAddress in - let match = memmem(baseAddress + startOffset, count - startOffset, needle.baseAddress, needle.count) - return match.map { match in - let start = baseAddress.distance(to: match.assumingMemoryBound(to: UInt8.self)) - return start ..+ needle.count - } + guard count > 0, let baseAddress else { + return nil + } + guard needle.count > 0, let needleBaseAddress = needle.baseAddress else { + return nil + } + guard let match = memmem(baseAddress + startOffset, count - startOffset, needleBaseAddress, needle.count) else { + return nil } + let start = baseAddress.distance(to: match.assumingMemoryBound(to: UInt8.self)) + return start ..+ needle.count } func rangeOf(bytes needle: [UInt8]) -> Range? { From 55abb1dec7e641488e4e22a3cfb77c4aa0301897 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Tue, 7 Jan 2025 17:20:55 +0100 Subject: [PATCH 03/18] Don't support thermal range throttling on non-macOS --- Sources/CompletionScoringTestSupport/TestExtensions.swift | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Sources/CompletionScoringTestSupport/TestExtensions.swift b/Sources/CompletionScoringTestSupport/TestExtensions.swift index ab890de23..4c2dc2a5c 100644 --- a/Sources/CompletionScoringTestSupport/TestExtensions.swift +++ b/Sources/CompletionScoringTestSupport/TestExtensions.swift @@ -65,6 +65,7 @@ extension XCTestCase { Thread.sleep(forTimeInterval: 2.0) } + #if canImport(Darwin) func induceThermalRange(_ range: ClosedRange) { var temperature: Int repeat { @@ -99,6 +100,7 @@ extension XCTestCase { return nil } }() + #endif private static let measurementsLogFile: String? = { UserDefaults.standard.string(forKey: "TestMeasurementLogPath") @@ -149,10 +151,14 @@ extension XCTestCase { let logFD = tryOrFailTest(try Self.openPerformanceLog(), message: "Failed to open performance log") var timings = Timings() for iteration in 0.. Int { var thermalLevel: UInt64 = 0 @@ -182,6 +189,7 @@ extension ProcessInfo { return Int(thermalLevel) } } +#endif extension String { fileprivate func dropSuffix(_ suffix: String) -> String { From 519408b1e29e7b1c4c0ebeb58d9fcac522c16009 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Tue, 7 Jan 2025 08:11:23 -0800 Subject: [PATCH 04/18] Import Linux C standard libaries in SKDResponse.swift --- Sources/SwiftSourceKitPlugin/SKDResponse.swift | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/Sources/SwiftSourceKitPlugin/SKDResponse.swift b/Sources/SwiftSourceKitPlugin/SKDResponse.swift index 661faf76b..6acd8d860 100644 --- a/Sources/SwiftSourceKitPlugin/SKDResponse.swift +++ b/Sources/SwiftSourceKitPlugin/SKDResponse.swift @@ -11,9 +11,20 @@ //===----------------------------------------------------------------------===// import Csourcekitd -import Darwin import SourceKitD +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#elseif canImport(Musl) +import Musl +#elseif canImport(CRT) +import CRT +#elseif canImport(Bionic) +import Bionic +#endif + final class SKDResponse: CustomStringConvertible, Sendable { enum ErrorKind { case connectionInterrupted From caa0223cf75c7a956a9512035683a63990a6453f Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Tue, 7 Jan 2025 17:21:12 +0100 Subject: [PATCH 05/18] Support platform-specific dynamic library file extensions --- Tests/SwiftSourceKitPluginTests/PluginPaths.swift | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/Tests/SwiftSourceKitPluginTests/PluginPaths.swift b/Tests/SwiftSourceKitPluginTests/PluginPaths.swift index fa13488b5..020fcf8ef 100644 --- a/Tests/SwiftSourceKitPluginTests/PluginPaths.swift +++ b/Tests/SwiftSourceKitPluginTests/PluginPaths.swift @@ -82,8 +82,15 @@ private func pluginPaths(relativeTo base: URL) -> PluginPaths? { // When building using 'swift test' do { - let clientPlugin = base.appendingPathComponent("libSwiftSourceKitClientPlugin.dylib") - let servicePlugin = base.appendingPathComponent("libSwiftSourceKitPlugin.dylib") + #if canImport(Darwin) + let dylibExtension = "dylib" + #elseif os(Windows) + let dylibExtension = "dll" + #else + let dylibExtension = "so" + #endif + let clientPlugin = base.appendingPathComponent("libSwiftSourceKitClientPlugin.\(dylibExtension)") + let servicePlugin = base.appendingPathComponent("libSwiftSourceKitPlugin.\(dylibExtension)") if fileExists(at: clientPlugin) && fileExists(at: servicePlugin) { return PluginPaths(clientPlugin: clientPlugin, servicePlugin: servicePlugin) } From 9e663dafc871a034c59a27cccc0b4977756db412 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Tue, 7 Jan 2025 08:14:19 -0800 Subject: [PATCH 06/18] Skip tests that are only supported on macOS --- .../SwiftSourceKitPluginTests.swift | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Tests/SwiftSourceKitPluginTests/SwiftSourceKitPluginTests.swift b/Tests/SwiftSourceKitPluginTests/SwiftSourceKitPluginTests.swift index 1645c73e7..fb44dfb10 100644 --- a/Tests/SwiftSourceKitPluginTests/SwiftSourceKitPluginTests.swift +++ b/Tests/SwiftSourceKitPluginTests/SwiftSourceKitPluginTests.swift @@ -993,6 +993,9 @@ final class SwiftSourceKitPluginTests: XCTestCase { // rdar://104381080 (NSImage(imageLiteralResourceName:) was my top completion — this seems odd) func testPopularityForTypeFromSubmodule() async throws { + #if !os(macOS) + try XCTSkipIf(true, "AppKit is only defined on macOS") + #endif let sourcekitd = try await getSourceKitD() let path = scratchFilePath() let positions = try await sourcekitd.openDocument( @@ -1507,6 +1510,9 @@ final class SwiftSourceKitPluginTests: XCTestCase { } func testCompletionDiagnostics() async throws { + #if !os(macOS) + try XCTSkipIf(true, "Soft deprecation is only defined for macOS in this test case") + #endif let sourcekitd = try await getSourceKitD() let path = scratchFilePath() let positions = try await sourcekitd.openDocument( From b78c91c295178099976fc2ae844096c9d7720fcb Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Tue, 7 Jan 2025 21:22:10 +0100 Subject: [PATCH 07/18] Avoid using `XCTestCase.invocation` This is not available on non-Darwin platforms. --- .../TestExtensions.swift | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/Sources/CompletionScoringTestSupport/TestExtensions.swift b/Sources/CompletionScoringTestSupport/TestExtensions.swift index 4c2dc2a5c..e49b5ca32 100644 --- a/Sources/CompletionScoringTestSupport/TestExtensions.swift +++ b/Sources/CompletionScoringTestSupport/TestExtensions.swift @@ -127,14 +127,6 @@ extension XCTestCase { } } - private var testName: String? { - self.invocation?.selector.description.dropPrefix("test").dropSuffix("AndReturnError:") - } - - private var testDisplayName: String { - return testName ?? "(Unnamed Test)" - } - func tryOrFailTest(_ expression: @autoclosure () throws -> R?, message: String) -> R? { do { return try expression() @@ -147,7 +139,7 @@ extension XCTestCase { /// Run `body()` `iterations`, gathering timing stats, and print them. /// In between runs, coax for the machine into an arbitrary but consistent thermal state by either sleeping or doing /// pointless work so that results are more comparable run to run, no matter else is happening on the machine. - package func gaugeTiming(iterations: Int = 1, _ body: () -> ()) { + package func gaugeTiming(iterations: Int = 1, testName: String = #function, _ body: () -> ()) { let logFD = tryOrFailTest(try Self.openPerformanceLog(), message: "Failed to open performance log") var timings = Timings() for iteration in 0.. Date: Tue, 7 Jan 2025 21:22:44 +0100 Subject: [PATCH 08/18] Communicate the path of sourcekitd that the plugin was loaded from to the plugin --- .../DynamicallyLoadedSourceKitD.swift | 6 +- Sources/SourceKitD/dlopen.swift | 4 -- .../ClientPlugin.swift | 42 +++++++++++++- Sources/SwiftSourceKitPlugin/Plugin.swift | 35 ++++++++++- .../CompletionResultsArray.swift | 6 +- ...llyLoadedSourceKitD+relativeToPlugin.swift | 58 ------------------- ...namicallyLoadedSourceKitdD+forPlugin.swift | 37 ++++++++++++ 7 files changed, 116 insertions(+), 72 deletions(-) delete mode 100644 Sources/SwiftSourceKitPluginCommon/DynamicallyLoadedSourceKitD+relativeToPlugin.swift create mode 100644 Sources/SwiftSourceKitPluginCommon/DynamicallyLoadedSourceKitdD+forPlugin.swift diff --git a/Sources/SourceKitD/DynamicallyLoadedSourceKitD.swift b/Sources/SourceKitD/DynamicallyLoadedSourceKitD.swift index a0c7b2eba..d49eb099d 100644 --- a/Sources/SourceKitD/DynamicallyLoadedSourceKitD.swift +++ b/Sources/SourceKitD/DynamicallyLoadedSourceKitD.swift @@ -111,18 +111,18 @@ package actor DynamicallyLoadedSourceKitD: SourceKitD { ) } - private init(dylib path: URL, pluginPaths: PluginPaths?) throws { + package init(dylib path: URL, pluginPaths: PluginPaths?, initialize: Bool = true) throws { #if os(Windows) let dlopenModes: DLOpenFlags = [] #else let dlopenModes: DLOpenFlags = [.lazy, .local, .first] #endif - let dlhandle = try dlopen(path.filePath, mode: []) + let dlhandle = try dlopen(path.filePath, mode: dlopenModes) try self.init( dlhandle: dlhandle, path: path, pluginPaths: pluginPaths, - initialize: true + initialize: initialize ) } diff --git a/Sources/SourceKitD/dlopen.swift b/Sources/SourceKitD/dlopen.swift index fe595e59a..3c402c487 100644 --- a/Sources/SourceKitD/dlopen.swift +++ b/Sources/SourceKitD/dlopen.swift @@ -34,10 +34,6 @@ package final class DLHandle: Sendable { #endif } - #if !os(Windows) - package static let rtldDefault = DLHandle(rawValue: Handle(handle: UnsafeMutableRawPointer(bitPattern: -2)!)) - #endif - fileprivate let rawValue: ThreadSafeBox fileprivate init(rawValue: Handle) { diff --git a/Sources/SwiftSourceKitClientPlugin/ClientPlugin.swift b/Sources/SwiftSourceKitClientPlugin/ClientPlugin.swift index 06a115c72..c326ff5c5 100644 --- a/Sources/SwiftSourceKitClientPlugin/ClientPlugin.swift +++ b/Sources/SwiftSourceKitClientPlugin/ClientPlugin.swift @@ -10,7 +10,9 @@ // //===----------------------------------------------------------------------===// +import Foundation import SourceKitD +import SwiftExtensionsForPlugin import SwiftSourceKitPluginCommon #if compiler(>=6) @@ -19,12 +21,46 @@ public import Csourcekitd import Csourcekitd #endif +/// Legacy plugin initialization logic in which sourcekitd does not inform the plugin about the sourcekitd path it was +/// loaded from. @_cdecl("sourcekitd_plugin_initialize") public func sourcekitd_plugin_initialize(_ params: sourcekitd_api_plugin_initialize_params_t) { - let skd = DynamicallyLoadedSourceKitD.relativeToPlugin - let customBufferStart = skd.pluginApi.plugin_initialize_custom_buffer_start(params) + #if canImport(Darwin) + var dlInfo = Dl_info() + dladdr(#dsohandle, &dlInfo) + let path = String(cString: dlInfo.dli_fname) + var url = URL(fileURLWithPath: path, isDirectory: false) + while url.pathExtension != "framework" && url.lastPathComponent != "/" { + url.deleteLastPathComponent() + } + url = + url + .deletingLastPathComponent() + .appendingPathComponent("sourcekitd.framework") + .appendingPathComponent("sourcekitd") + try! url.filePath.withCString { sourcekitdPath in + sourcekitd_plugin_initialize_2(params, sourcekitdPath) + } + #else + fatalError("sourcekitd_plugin_initialize is not supported on non-Darwin platforms") + #endif +} + +@_cdecl("sourcekitd_plugin_initialize_2") +public func sourcekitd_plugin_initialize_2( + _ params: sourcekitd_api_plugin_initialize_params_t, + _ sourcekitdPath: UnsafePointer +) { + DynamicallyLoadedSourceKitD.forPlugin = try! DynamicallyLoadedSourceKitD( + dylib: URL(fileURLWithPath: String(cString: sourcekitdPath)), + pluginPaths: nil, + initialize: false + ) + let sourcekitd = DynamicallyLoadedSourceKitD.forPlugin + + let customBufferStart = sourcekitd.pluginApi.plugin_initialize_custom_buffer_start(params) let arrayBuffKind = customBufferStart - skd.pluginApi.plugin_initialize_register_custom_buffer( + sourcekitd.pluginApi.plugin_initialize_register_custom_buffer( params, arrayBuffKind, CompletionResultsArray.arrayFuncs.rawValue diff --git a/Sources/SwiftSourceKitPlugin/Plugin.swift b/Sources/SwiftSourceKitPlugin/Plugin.swift index 4f65383f6..95fe8ea87 100644 --- a/Sources/SwiftSourceKitPlugin/Plugin.swift +++ b/Sources/SwiftSourceKitPlugin/Plugin.swift @@ -143,9 +143,42 @@ final class RequestHandler: Sendable { } } +/// Legacy plugin initialization logic in which sourcekitd does not inform the plugin about the sourcekitd path it was +/// loaded from. @_cdecl("sourcekitd_plugin_initialize") public func sourcekitd_plugin_initialize(_ params: sourcekitd_api_plugin_initialize_params_t) { - let sourcekitd = DynamicallyLoadedSourceKitD.relativeToPlugin + #if canImport(Darwin) + var dlInfo = Dl_info() + dladdr(#dsohandle, &dlInfo) + let path = String(cString: dlInfo.dli_fname) + var url = URL(fileURLWithPath: path, isDirectory: false) + while url.pathExtension != "framework" && url.lastPathComponent != "/" { + url.deleteLastPathComponent() + } + url = + url + .deletingLastPathComponent() + .appendingPathComponent("sourcekitd.framework") + .appendingPathComponent("sourcekitd") + try! url.filePath.withCString { sourcekitdPath in + sourcekitd_plugin_initialize_2(params, sourcekitdPath) + } + #else + fatalError("sourcekitd_plugin_initialize is not supported on non-Darwin platforms") + #endif +} + +@_cdecl("sourcekitd_plugin_initialize_2") +public func sourcekitd_plugin_initialize_2( + _ params: sourcekitd_api_plugin_initialize_params_t, + _ sourcekitdPath: UnsafePointer +) { + DynamicallyLoadedSourceKitD.forPlugin = try! DynamicallyLoadedSourceKitD( + dylib: URL(fileURLWithPath: String(cString: parentLibraryPath)), + pluginPaths: nil, + initialize: false + ) + let sourcekitd = DynamicallyLoadedSourceKitD.forPlugin let completionResultsBufferKind = sourcekitd.pluginApi.plugin_initialize_custom_buffer_start(params) let isClientOnly = sourcekitd.pluginApi.plugin_initialize_is_client_only(params) diff --git a/Sources/SwiftSourceKitPluginCommon/CompletionResultsArray.swift b/Sources/SwiftSourceKitPluginCommon/CompletionResultsArray.swift index dce46563e..17ea1ca43 100644 --- a/Sources/SwiftSourceKitPluginCommon/CompletionResultsArray.swift +++ b/Sources/SwiftSourceKitPluginCommon/CompletionResultsArray.swift @@ -126,7 +126,7 @@ package struct CompletionResultsArray { } package static let arrayFuncs: VariantFunctions = { - let sourcekitd = DynamicallyLoadedSourceKitD.relativeToPlugin + let sourcekitd = DynamicallyLoadedSourceKitD.forPlugin let funcs = sourcekitd.pluginApi.variant_functions_create()! sourcekitd.pluginApi.variant_functions_set_get_type(funcs, { _ in SOURCEKITD_API_VARIANT_TYPE_ARRAY }) sourcekitd.pluginApi.variant_functions_set_array_get_count(funcs, { arrayGetCount($0) }) @@ -156,7 +156,7 @@ package struct CompletionResultsArray { } } - let sourcekitd = DynamicallyLoadedSourceKitD.relativeToPlugin + let sourcekitd = DynamicallyLoadedSourceKitD.forPlugin let keys = sourcekitd.keys let results = CompletionResultsArray(dict) @@ -199,7 +199,7 @@ package struct CompletionResultsArray { } static let dictionaryFuncs: VariantFunctions = { - let sourcekitd = DynamicallyLoadedSourceKitD.relativeToPlugin + let sourcekitd = DynamicallyLoadedSourceKitD.forPlugin let funcs = sourcekitd.pluginApi.variant_functions_create()! sourcekitd.pluginApi.variant_functions_set_get_type(funcs, { _ in SOURCEKITD_API_VARIANT_TYPE_DICTIONARY }) sourcekitd.pluginApi.variant_functions_set_dictionary_apply(funcs, { dictionaryApply($0, $1, $2) }) diff --git a/Sources/SwiftSourceKitPluginCommon/DynamicallyLoadedSourceKitD+relativeToPlugin.swift b/Sources/SwiftSourceKitPluginCommon/DynamicallyLoadedSourceKitD+relativeToPlugin.swift deleted file mode 100644 index eb84b840a..000000000 --- a/Sources/SwiftSourceKitPluginCommon/DynamicallyLoadedSourceKitD+relativeToPlugin.swift +++ /dev/null @@ -1,58 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// - -import Foundation -import SKLogging -import SwiftExtensions - -#if compiler(>=6) -package import SourceKitD -#else -import SourceKitD -#endif - -extension DynamicallyLoadedSourceKitD { - package static let relativeToPlugin: SourceKitD = { - var dlInfo = Dl_info() - dladdr(#dsohandle, &dlInfo) - let path = String(cString: dlInfo.dli_fname) - var url = URL(fileURLWithPath: path, isDirectory: false) - while url.pathExtension != "framework" && url.lastPathComponent != "/" { - url.deleteLastPathComponent() - } - url = - url - .deletingLastPathComponent() - .appendingPathComponent("sourcekitd.framework") - .appendingPathComponent("sourcekitd") - - let dlhandle: DLHandle - do { - dlhandle = try dlopen(url.filePath, mode: [.local, .first]) - } catch { - Logger(subsystem: "org.swift.sourcekit.client-plugin", category: "Loading") - .error("failed to find sourcekitd.framework relative to \(path); falling back to RTLD_DEFAULT") - dlhandle = .rtldDefault - } - - do { - return try DynamicallyLoadedSourceKitD( - dlhandle: dlhandle, - path: URL(string: "fake://")!, - pluginPaths: nil, - initialize: false - ) - } catch { - fatalError("Failed to load sourcekitd: \(error)") - } - }() -} diff --git a/Sources/SwiftSourceKitPluginCommon/DynamicallyLoadedSourceKitdD+forPlugin.swift b/Sources/SwiftSourceKitPluginCommon/DynamicallyLoadedSourceKitdD+forPlugin.swift new file mode 100644 index 000000000..3b9f340b9 --- /dev/null +++ b/Sources/SwiftSourceKitPluginCommon/DynamicallyLoadedSourceKitdD+forPlugin.swift @@ -0,0 +1,37 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation +import SKLogging +import SwiftExtensions + +#if compiler(>=6) +package import SourceKitD +#else +import SourceKitD +#endif + +extension DynamicallyLoadedSourceKitD { + private static nonisolated(unsafe) var _forPlugin: SourceKitD? + package static var forPlugin: SourceKitD { + get { + guard let _forPlugin else { + fatalError("forPlugin must only be accessed after it was set in sourcekitd_plugin_initialize_2") + } + return _forPlugin + } + set { + precondition(_forPlugin == nil, "DynamicallyLoadedSourceKitD.forPlugin must not be set twice") + _forPlugin = newValue + } + } +} From 1d32512ed03e6753a8c9c607cf52a12b5d9069fc Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Tue, 7 Jan 2025 21:23:03 +0100 Subject: [PATCH 09/18] Fix issue that caused code completion request to not be cancellable --- .../CodeCompletion/Connection.swift | 16 ++++++++++++---- .../CompletionProvider.swift | 12 +++++------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/Sources/SwiftSourceKitPlugin/CodeCompletion/Connection.swift b/Sources/SwiftSourceKitPlugin/CodeCompletion/Connection.swift index 272cb3a6f..457baad7e 100644 --- a/Sources/SwiftSourceKitPlugin/CodeCompletion/Connection.swift +++ b/Sources/SwiftSourceKitPlugin/CodeCompletion/Connection.swift @@ -113,6 +113,18 @@ final class Connection { sourcekitd.ideApi.connection_dispose(impl) } + //// A function that can be called to cancel a request with a request. + /// + /// This is not a member function on `Connection` so that `CompletionProvider` can store + /// this closure in a member and call it even while the `CompletionProvider` actor is busy + /// fulfilling a completion request and thus can't access `connection`. + var cancellationFunc: @Sendable (RequestHandle) -> Void { + nonisolated(unsafe) let impl = self.impl + return { [sourcekitd] handle in + sourcekitd.ideApi.cancel_request(impl, handle.handle) + } + } + func openDocument(path: String, contents: String, compilerArguments: [String]? = nil) { if documents[path] != nil { logger.error("Document at '\(path)' is already open") @@ -219,10 +231,6 @@ final class Connection { ) } - func cancel(handle: RequestHandle) { - sourcekitd.ideApi.cancel_request(impl, handle.handle) - } - func markCachedCompilerInstanceShouldBeInvalidated() { sourcekitd.ideApi.connection_mark_cached_compiler_instance_should_be_invalidated(impl, nil) } diff --git a/Sources/SwiftSourceKitPlugin/CompletionProvider.swift b/Sources/SwiftSourceKitPlugin/CompletionProvider.swift index cf2b457b8..d02ed6d7b 100644 --- a/Sources/SwiftSourceKitPlugin/CompletionProvider.swift +++ b/Sources/SwiftSourceKitPlugin/CompletionProvider.swift @@ -86,6 +86,9 @@ actor CompletionProvider { private let connection: Connection + /// See `Connection.cancellationFunc` + private nonisolated let cancel: @Sendable (RequestHandle) -> Void + /// The XPC custom buffer kind for `CompletionResultsArray` private let completionResultsBufferKind: UInt64 @@ -101,17 +104,12 @@ actor CompletionProvider { opaqueIDEInspectionInstance: opaqueIDEInspectionInstance?.value, sourcekitd: sourcekitd ) + self.cancel = connection.cancellationFunc self.completionResultsBufferKind = completionResultsBufferKind } nonisolated func cancel(handle: RequestHandle) { - Task { - await self.cancelImpl(handle: handle) - } - } - - private func cancelImpl(handle: RequestHandle) { - connection.cancel(handle: handle) + self.cancel(handle) } func handleDocumentOpen(_ request: SKDRequestDictionaryReader) { From 150423b490039dcc8390abddd2793653becd07c8 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Wed, 8 Jan 2025 09:36:17 +0100 Subject: [PATCH 10/18] Fix test failures --- .../Utilities/SwiftExtensions.swift | 12 ++++++++++++ Sources/ToolchainRegistry/Toolchain.swift | 2 +- Utilities/build-script-helper.py | 7 +++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/Sources/CompletionScoring/Utilities/SwiftExtensions.swift b/Sources/CompletionScoring/Utilities/SwiftExtensions.swift index a8e95c118..fdea664fc 100644 --- a/Sources/CompletionScoring/Utilities/SwiftExtensions.swift +++ b/Sources/CompletionScoring/Utilities/SwiftExtensions.swift @@ -12,6 +12,18 @@ import Foundation +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#elseif canImport(Musl) +import Musl +#elseif canImport(CRT) +import CRT +#elseif canImport(Bionic) +import Bionic +#endif + extension Range where Bound: Numeric { init(from: Bound, length: Bound) { self = from..<(from + length) diff --git a/Sources/ToolchainRegistry/Toolchain.swift b/Sources/ToolchainRegistry/Toolchain.swift index e06a63cf6..24f1993f3 100644 --- a/Sources/ToolchainRegistry/Toolchain.swift +++ b/Sources/ToolchainRegistry/Toolchain.swift @@ -347,7 +347,7 @@ func containingXCToolchain( var path = path while !path.isRoot { if path.pathExtension == "xctoolchain" { - if let infoPlist = orLog("", { try XCToolchainPlist(fromDirectory: path) }) { + if let infoPlist = orLog("Loading information from xctoolchain", { try XCToolchainPlist(fromDirectory: path) }) { return (infoPlist, path) } return nil diff --git a/Utilities/build-script-helper.py b/Utilities/build-script-helper.py index 6c7ea7e5e..9f7fc615c 100755 --- a/Utilities/build-script-helper.py +++ b/Utilities/build-script-helper.py @@ -216,6 +216,11 @@ def run_tests(swift_exec: str, args: argparse.Namespace) -> None: print('Cleaning ' + tests) shutil.rmtree(tests, ignore_errors=True) + # Build the plugin so it can be used by the tests. SwiftPM is not able to express a dependency from a test target on + # a product. + build_single_product('SwiftSourceKitPlugin', swift_exec, args) + build_single_product('SwiftSourceKitClientPlugin', swift_exec, args) + cmd = [ swift_exec, 'test', '--disable-testable-imports', @@ -265,6 +270,8 @@ def handle_invocation(swift_exec: str, args: argparse.Namespace) -> None: if args.action == 'build': build_single_product("sourcekit-lsp", swift_exec, args) + build_single_product('SwiftSourceKitPlugin', swift_exec, args) + build_single_product('SwiftSourceKitClientPlugin', swift_exec, args) elif args.action == 'test': run_tests(swift_exec, args) elif args.action == 'install': From e5f1bbfc03470fe06243a46bf48f9c959e677981 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Wed, 8 Jan 2025 22:25:16 +0100 Subject: [PATCH 11/18] Set plugin paths in all tests Otherwise, we were trying to load sourcekitd without the plugins for the normal SourceKIt-LSP tests and with the plugins for the plugin tests. --- .../RunSourcekitdRequestCommand.swift | 3 +- Sources/SKOptions/SourceKitLSPOptions.swift | 30 ++++++++++ .../SKTestSupport}/PluginPaths.swift | 40 +++++++++---- .../TestSourceKitLSPClient.swift | 6 ++ .../DynamicallyLoadedSourceKitD.swift | 2 +- Sources/SourceKitD/SourceKitD.swift | 4 +- .../Swift/SwiftLanguageService.swift | 16 ++++- Sources/SwiftSourceKitPlugin/Plugin.swift | 60 ++++++++++++++++++- Tests/DiagnoseTests/DiagnoseTests.swift | 5 +- Tests/SourceKitDTests/SourceKitDTests.swift | 5 +- Tests/SourceKitLSPTests/LifecycleTests.swift | 9 ++- .../SwiftSourceKitPluginTests.swift | 22 ++----- 12 files changed, 162 insertions(+), 40 deletions(-) rename {Tests/SwiftSourceKitPluginTests => Sources/SKTestSupport}/PluginPaths.swift (78%) diff --git a/Sources/Diagnose/RunSourcekitdRequestCommand.swift b/Sources/Diagnose/RunSourcekitdRequestCommand.swift index db383307a..8ed6f7569 100644 --- a/Sources/Diagnose/RunSourcekitdRequestCommand.swift +++ b/Sources/Diagnose/RunSourcekitdRequestCommand.swift @@ -67,7 +67,8 @@ package struct RunSourceKitdRequestCommand: AsyncParsableCommand { throw ExitCode(1) } let sourcekitd = try await DynamicallyLoadedSourceKitD.getOrCreate( - dylibPath: sourcekitdPath + dylibPath: sourcekitdPath, + pluginPaths: nil ) var lastResponse: SKDResponse? diff --git a/Sources/SKOptions/SourceKitLSPOptions.swift b/Sources/SKOptions/SourceKitLSPOptions.swift index 03e9fd478..5d5addbfa 100644 --- a/Sources/SKOptions/SourceKitLSPOptions.swift +++ b/Sources/SKOptions/SourceKitLSPOptions.swift @@ -242,6 +242,26 @@ public struct SourceKitLSPOptions: Sendable, Codable, Equatable { } } + public struct SourceKitDOptions: Sendable, Codable, Equatable { + /// When set, load the SourceKit client plugin from this path instead of locating it inside the toolchain. + public var clientPlugin: String? + + /// When set, load the SourceKit service plugin from this path instead of locating it inside the toolchain. + public var servicePlugin: String? + + public init(clientPlugin: String? = nil, servicePlugin: String? = nil) { + self.clientPlugin = clientPlugin + self.servicePlugin = servicePlugin + } + + static func merging(base: SourceKitDOptions, override: SourceKitDOptions?) -> SourceKitDOptions { + return SourceKitDOptions( + clientPlugin: override?.clientPlugin ?? base.clientPlugin, + servicePlugin: override?.servicePlugin ?? base.servicePlugin + ) + } + } + public enum BackgroundPreparationMode: String, Sendable, Codable, Equatable { /// Build a target to prepare it. case build @@ -303,6 +323,13 @@ public struct SourceKitLSPOptions: Sendable, Codable, Equatable { set { logging = newValue } } + /// Options modifying the behavior of sourcekitd. + private var sourcekitd: SourceKitDOptions? + public var sourcekitdOrDefault: SourceKitDOptions { + get { sourcekitd ?? .init() } + set { sourcekitd = newValue } + } + /// Default workspace type. Overrides workspace type selection logic. public var defaultWorkspaceType: WorkspaceType? /// Directory in which generated interfaces and macro expansions should be stored. @@ -381,6 +408,7 @@ public struct SourceKitLSPOptions: Sendable, Codable, Equatable { clangdOptions: [String]? = nil, index: IndexOptions? = .init(), logging: LoggingOptions? = .init(), + sourcekitd: SourceKitDOptions? = .init(), defaultWorkspaceType: WorkspaceType? = nil, generatedFilesPath: String? = nil, backgroundIndexing: Bool? = nil, @@ -398,6 +426,7 @@ public struct SourceKitLSPOptions: Sendable, Codable, Equatable { self.clangdOptions = clangdOptions self.index = index self.logging = logging + self.sourcekitd = sourcekitd self.generatedFilesPath = generatedFilesPath self.defaultWorkspaceType = defaultWorkspaceType self.backgroundIndexing = backgroundIndexing @@ -454,6 +483,7 @@ public struct SourceKitLSPOptions: Sendable, Codable, Equatable { clangdOptions: override?.clangdOptions ?? base.clangdOptions, index: IndexOptions.merging(base: base.indexOrDefault, override: override?.index), logging: LoggingOptions.merging(base: base.loggingOrDefault, override: override?.logging), + sourcekitd: SourceKitDOptions.merging(base: base.sourcekitdOrDefault, override: override?.sourcekitd), defaultWorkspaceType: override?.defaultWorkspaceType ?? base.defaultWorkspaceType, generatedFilesPath: override?.generatedFilesPath ?? base.generatedFilesPath, backgroundIndexing: override?.backgroundIndexing ?? base.backgroundIndexing, diff --git a/Tests/SwiftSourceKitPluginTests/PluginPaths.swift b/Sources/SKTestSupport/PluginPaths.swift similarity index 78% rename from Tests/SwiftSourceKitPluginTests/PluginPaths.swift rename to Sources/SKTestSupport/PluginPaths.swift index 020fcf8ef..88a0dd494 100644 --- a/Tests/SwiftSourceKitPluginTests/PluginPaths.swift +++ b/Sources/SKTestSupport/PluginPaths.swift @@ -11,7 +11,12 @@ //===----------------------------------------------------------------------===// import Foundation + +#if compiler(>=6) +package import SourceKitD +#else import SourceKitD +#endif /// The path to the `SwiftSourceKitPluginTests` test bundle. This gives us a hook into the the build directory. private let xctestBundle: URL = { @@ -29,7 +34,7 @@ private let xctestBundle: URL = { }() /// When running tests from Xcode, determine the build configuration of the package. -var inferedXcodeBuildConfiguration: String? { +var inferredXcodeBuildConfiguration: String? { if let xcodeBuildDirectory = ProcessInfo.processInfo.environment["__XCODE_BUILT_PRODUCTS_DIR_PATHS"] { return URL(fileURLWithPath: xcodeBuildDirectory).lastPathComponent } else { @@ -47,7 +52,7 @@ private func fileExists(at url: URL) -> Bool { /// Implementation detail of `sourceKitPluginPaths` which walks up the directory structure, repeatedly calling this method. private func pluginPaths(relativeTo base: URL) -> PluginPaths? { // When building in Xcode - if let buildConfiguration = inferedXcodeBuildConfiguration { + if let buildConfiguration = inferredXcodeBuildConfiguration { let frameworksDir = base.appendingPathComponent("Products") .appendingPathComponent(buildConfiguration) .appendingPathComponent("PackageFrameworks") @@ -99,16 +104,25 @@ private func pluginPaths(relativeTo base: URL) -> PluginPaths? { return nil } -/// Returns the path the the client plugin and the server plugin within the current build directory. -/// -/// Returns `nil` if either of the plugins can't be found in the build directory. -let sourceKitPluginPaths: PluginPaths? = { - var base = xctestBundle - while base.pathComponents.count > 1 { - if let paths = pluginPaths(relativeTo: base) { - return paths +/// Returns the paths from which the SourceKit plugins should be loaded or throws an error if the plugins cannot be +/// found. +package var sourceKitPluginPaths: PluginPaths { + get throws { + struct PluginLoadingError: Error, CustomStringConvertible { + var description: String = + "Could not find SourceKit plugin. Ensure that you build the entire SourceKit-LSP package before running tests." + } + + var base = xctestBundle + while base.pathComponents.count > 1 { + if let paths = pluginPaths(relativeTo: base) { + return paths + } + base = base.deletingLastPathComponent() } - base = base.deletingLastPathComponent() + + // If we couldn't find the plugins, keep `didLoadPlugin = false`, which will throw an error in each test case's + // `setUp` function. We can't throw an error from the class `setUp` function. + throw PluginLoadingError() } - return nil -}() +} diff --git a/Sources/SKTestSupport/TestSourceKitLSPClient.swift b/Sources/SKTestSupport/TestSourceKitLSPClient.swift index b6cb8a3f8..fc31ce7bb 100644 --- a/Sources/SKTestSupport/TestSourceKitLSPClient.swift +++ b/Sources/SKTestSupport/TestSourceKitLSPClient.swift @@ -18,6 +18,7 @@ import LanguageServerProtocolJSONRPC import LanguageServerProtocolExtensions package import SKOptions import SKUtilities +import SourceKitD package import SourceKitLSP import SwiftExtensions import SwiftSyntax @@ -31,6 +32,7 @@ import LanguageServerProtocolJSONRPC import LanguageServerProtocolExtensions import SKOptions import SKUtilities +import SourceKitD import SourceKitLSP import SwiftExtensions import SwiftSyntax @@ -44,6 +46,10 @@ extension SourceKitLSPOptions { experimentalFeatures: Set? = nil ) -> SourceKitLSPOptions { return SourceKitLSPOptions( + sourcekitd: SourceKitDOptions( + clientPlugin: try! sourceKitPluginPaths.clientPlugin.filePath, + servicePlugin: try! sourceKitPluginPaths.servicePlugin.filePath + ), backgroundIndexing: backgroundIndexing, experimentalFeatures: experimentalFeatures, swiftPublishDiagnosticsDebounceDuration: 0, diff --git a/Sources/SourceKitD/DynamicallyLoadedSourceKitD.swift b/Sources/SourceKitD/DynamicallyLoadedSourceKitD.swift index d49eb099d..1ec6714fa 100644 --- a/Sources/SourceKitD/DynamicallyLoadedSourceKitD.swift +++ b/Sources/SourceKitD/DynamicallyLoadedSourceKitD.swift @@ -101,7 +101,7 @@ package actor DynamicallyLoadedSourceKitD: SourceKitD { package static func getOrCreate( dylibPath: URL, - pluginPaths: PluginPaths? = nil + pluginPaths: PluginPaths? ) async throws -> SourceKitD { try await SourceKitDRegistry.shared .getOrAdd( diff --git a/Sources/SourceKitD/SourceKitD.swift b/Sources/SourceKitD/SourceKitD.swift index 8a51b76ee..2657a531d 100644 --- a/Sources/SourceKitD/SourceKitD.swift +++ b/Sources/SourceKitD/SourceKitD.swift @@ -30,8 +30,8 @@ fileprivate struct SourceKitDRequestHandle: Sendable { } package struct PluginPaths: Equatable, CustomLogStringConvertible { - let clientPlugin: URL - let servicePlugin: URL + package let clientPlugin: URL + package let servicePlugin: URL package init(clientPlugin: URL, servicePlugin: URL) { self.clientPlugin = clientPlugin diff --git a/Sources/SourceKitLSP/Swift/SwiftLanguageService.swift b/Sources/SourceKitLSP/Swift/SwiftLanguageService.swift index fa86f15c9..28e824236 100644 --- a/Sources/SourceKitLSP/Swift/SwiftLanguageService.swift +++ b/Sources/SourceKitLSP/Swift/SwiftLanguageService.swift @@ -208,7 +208,21 @@ package actor SwiftLanguageService: LanguageService, Sendable { guard let sourcekitd = toolchain.sourcekitd else { return nil } self.sourceKitLSPServer = sourceKitLSPServer self.swiftFormat = toolchain.swiftFormat - self.sourcekitd = try await DynamicallyLoadedSourceKitD.getOrCreate(dylibPath: sourcekitd) + let pluginPaths: PluginPaths? + if let clientPlugin = options.sourcekitdOrDefault.clientPlugin, + let servicePlugin = options.sourcekitdOrDefault.servicePlugin + { + pluginPaths = PluginPaths( + clientPlugin: URL(fileURLWithPath: clientPlugin), + servicePlugin: URL(fileURLWithPath: servicePlugin) + ) + } else { + pluginPaths = nil + } + self.sourcekitd = try await DynamicallyLoadedSourceKitD.getOrCreate( + dylibPath: sourcekitd, + pluginPaths: pluginPaths + ) self.capabilityRegistry = workspace.capabilityRegistry self.semanticIndexManager = workspace.semanticIndexManager self.testHooks = testHooks diff --git a/Sources/SwiftSourceKitPlugin/Plugin.swift b/Sources/SwiftSourceKitPlugin/Plugin.swift index 95fe8ea87..f6b33f131 100644 --- a/Sources/SwiftSourceKitPlugin/Plugin.swift +++ b/Sources/SwiftSourceKitPlugin/Plugin.swift @@ -168,16 +168,74 @@ public func sourcekitd_plugin_initialize(_ params: sourcekitd_api_plugin_initial #endif } +#if canImport(Darwin) +private extension DynamicallyLoadedSourceKitD { + /// When a plugin is initialized, it gets passed the library it was loaded from to `sourcekitd_plugin_initialize_2`. + /// + /// Since the plugin wants to interact with sourcekitd in-process, it needs to load `sourcekitdInProc`. This function + /// loads `sourcekitdInProc` relative to the parent library path, if it exists, or `sourcekitd` if `sourcekitdInProc` + /// doesn't exist (eg. on Linux where `sourcekitd` is already in-process). + static func inProcLibrary(relativeTo parentLibraryPath: URL) throws -> DynamicallyLoadedSourceKitD { + var frameworkUrl = parentLibraryPath + + // Remove path components until we reach the `sourcekitd.framework` directory. The plugin might have been loaded + // from an XPC service, in which case `parentLibraryPath` is + // `sourcekitd.framework/XPCServices/SourceKitService.xpc/Contents/MacOS/SourceKitService`. + while frameworkUrl.pathExtension != "framework" { + guard frameworkUrl.pathComponents.count > 1 else { + struct NoFrameworkPathError: Error, CustomStringConvertible { + var parentLibraryPath: URL + var description: String { "Could not find .framework directory relative to '\(parentLibraryPath)'" } + } + throw NoFrameworkPathError(parentLibraryPath: parentLibraryPath) + } + frameworkUrl.deleteLastPathComponent() + } + frameworkUrl.deleteLastPathComponent() + + let inProcUrl = + frameworkUrl + .appendingPathComponent("sourcekitdInProc.framework") + .appendingPathComponent("sourcekitdInProc") + if FileManager.default.fileExists(at: inProcUrl) { + return try DynamicallyLoadedSourceKitD( + dylib: inProcUrl, + pluginPaths: nil, + initialize: false + ) + } + + let sourcekitdUrl = + frameworkUrl + .appendingPathComponent("sourcekitd.framework") + .appendingPathComponent("sourcekitd") + return try DynamicallyLoadedSourceKitD( + dylib: sourcekitdUrl, + pluginPaths: nil, + initialize: false + ) + } +} +#endif + @_cdecl("sourcekitd_plugin_initialize_2") public func sourcekitd_plugin_initialize_2( _ params: sourcekitd_api_plugin_initialize_params_t, - _ sourcekitdPath: UnsafePointer + _ parentLibraryPath: UnsafePointer ) { + #if canImport(Darwin) + // On macOS, we need to find sourcekitdInProc relative to the library the plugin was loaded from. + DynamicallyLoadedSourceKitD.forPlugin = try! DynamicallyLoadedSourceKitD.inProcLibrary( + relativeTo: URL(fileURLWithPath: String(cString: parentLibraryPath)) + ) + #else + // On other platforms, sourcekitd is always in process, so we can load it straight away. DynamicallyLoadedSourceKitD.forPlugin = try! DynamicallyLoadedSourceKitD( dylib: URL(fileURLWithPath: String(cString: parentLibraryPath)), pluginPaths: nil, initialize: false ) + #endif let sourcekitd = DynamicallyLoadedSourceKitD.forPlugin let completionResultsBufferKind = sourcekitd.pluginApi.plugin_initialize_custom_buffer_start(params) diff --git a/Tests/DiagnoseTests/DiagnoseTests.swift b/Tests/DiagnoseTests/DiagnoseTests.swift index 9e7ced6c5..31683e5eb 100644 --- a/Tests/DiagnoseTests/DiagnoseTests.swift +++ b/Tests/DiagnoseTests/DiagnoseTests.swift @@ -317,7 +317,10 @@ private class InProcessSourceKitRequestExecutor: SourceKitRequestExecutor { let requestString = try request.request(for: temporarySourceFile) logger.info("Sending request: \(requestString)") - let sourcekitd = try await DynamicallyLoadedSourceKitD.getOrCreate(dylibPath: sourcekitd) + let sourcekitd = try await DynamicallyLoadedSourceKitD.getOrCreate( + dylibPath: sourcekitd, + pluginPaths: sourceKitPluginPaths + ) let response = try await sourcekitd.run(requestYaml: requestString) logger.info("Received response: \(response.description)") diff --git a/Tests/SourceKitDTests/SourceKitDTests.swift b/Tests/SourceKitDTests/SourceKitDTests.swift index 8ab932290..98c121af4 100644 --- a/Tests/SourceKitDTests/SourceKitDTests.swift +++ b/Tests/SourceKitDTests/SourceKitDTests.swift @@ -26,7 +26,10 @@ import class TSCBasic.Process final class SourceKitDTests: XCTestCase { func testMultipleNotificationHandlers() async throws { let sourcekitdPath = await ToolchainRegistry.forTesting.default!.sourcekitd! - let sourcekitd = try await DynamicallyLoadedSourceKitD.getOrCreate(dylibPath: sourcekitdPath) + let sourcekitd = try await DynamicallyLoadedSourceKitD.getOrCreate( + dylibPath: sourcekitdPath, + pluginPaths: sourceKitPluginPaths + ) let keys = sourcekitd.keys let path = DocumentURI(for: .swift).pseudoPath diff --git a/Tests/SourceKitLSPTests/LifecycleTests.swift b/Tests/SourceKitLSPTests/LifecycleTests.swift index fef386c80..d6c625c57 100644 --- a/Tests/SourceKitLSPTests/LifecycleTests.swift +++ b/Tests/SourceKitLSPTests/LifecycleTests.swift @@ -45,7 +45,14 @@ final class LifecycleTests: XCTestCase { // Check that none of the keys in `SourceKitLSPOptions` are required. XCTAssertEqual( try JSONDecoder().decode(SourceKitLSPOptions.self, from: XCTUnwrap("{}".data(using: .utf8))), - SourceKitLSPOptions(swiftPM: nil, fallbackBuildSystem: nil, compilationDatabase: nil, index: nil, logging: nil) + SourceKitLSPOptions( + swiftPM: nil, + fallbackBuildSystem: nil, + compilationDatabase: nil, + index: nil, + logging: nil, + sourcekitd: nil + ) ) } diff --git a/Tests/SwiftSourceKitPluginTests/SwiftSourceKitPluginTests.swift b/Tests/SwiftSourceKitPluginTests/SwiftSourceKitPluginTests.swift index fb44dfb10..38b4ec900 100644 --- a/Tests/SwiftSourceKitPluginTests/SwiftSourceKitPluginTests.swift +++ b/Tests/SwiftSourceKitPluginTests/SwiftSourceKitPluginTests.swift @@ -20,23 +20,6 @@ import ToolchainRegistry import XCTest final class SwiftSourceKitPluginTests: XCTestCase { - /// Returns the paths from which the SourceKit plugins should be loaded or throws an error if the plugins cannot be - /// found. - private var pluginPaths: PluginPaths { - get throws { - struct PluginLoadingError: Error, CustomStringConvertible { - var description: String = "Could not find SourceKit plugin" - } - - guard let sourceKitPluginPaths else { - // If we couldn't find the plugins, keep `didLoadPlugin = false`, which will throw an error in each test case's - // `setUp` function. We can't throw an error from the class `setUp` function. - throw PluginLoadingError() - } - return sourceKitPluginPaths - } - } - /// Returns a path to a file name that is unique to this test execution. /// /// The file does not actually exist on disk. @@ -51,7 +34,10 @@ final class SwiftSourceKitPluginTests: XCTestCase { } throw NoSourceKitdFound() } - return try await DynamicallyLoadedSourceKitD.getOrCreate(dylibPath: sourcekitd, pluginPaths: try self.pluginPaths) + return try await DynamicallyLoadedSourceKitD.getOrCreate( + dylibPath: sourcekitd, + pluginPaths: try sourceKitPluginPaths + ) } func testBasicCompletion() async throws { From 2d3f933e63ba22b5ba16b83de7fca0e9417ca02c Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Wed, 8 Jan 2025 22:25:43 +0100 Subject: [PATCH 12/18] Close `DLHandle` if initializing sourcekitd fails --- .../DynamicallyLoadedSourceKitD.swift | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/Sources/SourceKitD/DynamicallyLoadedSourceKitD.swift b/Sources/SourceKitD/DynamicallyLoadedSourceKitD.swift index 1ec6714fa..17782b3a9 100644 --- a/Sources/SourceKitD/DynamicallyLoadedSourceKitD.swift +++ b/Sources/SourceKitD/DynamicallyLoadedSourceKitD.swift @@ -118,12 +118,17 @@ package actor DynamicallyLoadedSourceKitD: SourceKitD { let dlopenModes: DLOpenFlags = [.lazy, .local, .first] #endif let dlhandle = try dlopen(path.filePath, mode: dlopenModes) - try self.init( - dlhandle: dlhandle, - path: path, - pluginPaths: pluginPaths, - initialize: initialize - ) + do { + try self.init( + dlhandle: dlhandle, + path: path, + pluginPaths: pluginPaths, + initialize: initialize + ) + } catch { + try? dlhandle.close() + throw error + } } package init(dlhandle: DLHandle, path: URL, pluginPaths: PluginPaths?, initialize: Bool) throws { From 1c02ed68b1d40686d815bb2107a8683f5a17001f Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Wed, 8 Jan 2025 22:26:03 +0100 Subject: [PATCH 13/18] Only call `sourcekitd_register_plugin_path` if we have plugin paths --- Sources/SourceKitD/DynamicallyLoadedSourceKitD.swift | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Sources/SourceKitD/DynamicallyLoadedSourceKitD.swift b/Sources/SourceKitD/DynamicallyLoadedSourceKitD.swift index 17782b3a9..6a3e530d5 100644 --- a/Sources/SourceKitD/DynamicallyLoadedSourceKitD.swift +++ b/Sources/SourceKitD/DynamicallyLoadedSourceKitD.swift @@ -145,7 +145,9 @@ package actor DynamicallyLoadedSourceKitD: SourceKitD { self.pluginApiResult = Result(catching: { try sourcekitd_plugin_api_functions_t(dlhandle) }) self.servicePluginApiResult = Result(catching: { try sourcekitd_service_plugin_api_functions_t(dlhandle) }) - api.register_plugin_path?(pluginPaths?.clientPlugin.path, pluginPaths?.servicePlugin.path) + if let pluginPaths { + api.register_plugin_path?(pluginPaths.clientPlugin.path, pluginPaths.servicePlugin.path) + } if initialize { self.api.initialize() } From ab8bc45095957ca7a78ef50f1596b8225a1aaee3 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Wed, 8 Jan 2025 22:42:28 +0100 Subject: [PATCH 14/18] Define `memmem` using a header file On Ubuntu 20.04, you need to define `_GNU_SOURCE` before importing `` to get `memmem`. It is thus not available when importing `Glibc`. --- Package.swift | 9 ++++++-- .../include/CCompletionScoring.h | 23 +++++++++++++++++++ .../Utilities/SwiftExtensions.swift | 17 ++++---------- 3 files changed, 34 insertions(+), 15 deletions(-) create mode 100644 Sources/CCompletionScoring/include/CCompletionScoring.h diff --git a/Package.swift b/Package.swift index 00515fd01..9889292ed 100644 --- a/Package.swift +++ b/Package.swift @@ -111,17 +111,22 @@ var targets: [Target] = [ dependencies: [] ), + .target( + name: "CCompletionScoring", + dependencies: [] + ), + // MARK: CompletionScoring .target( name: "CompletionScoring", - dependencies: [], + dependencies: ["CCompletionScoring"], swiftSettings: globalSwiftSettings ), .target( name: "CompletionScoringForPlugin", - dependencies: [], + dependencies: ["CCompletionScoring"], swiftSettings: globalSwiftSettings ), diff --git a/Sources/CCompletionScoring/include/CCompletionScoring.h b/Sources/CCompletionScoring/include/CCompletionScoring.h new file mode 100644 index 000000000..875c55c9f --- /dev/null +++ b/Sources/CCompletionScoring/include/CCompletionScoring.h @@ -0,0 +1,23 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +#ifndef SOURCEKITLSP_CCOMPLETIONSCORING_H +#define SOURCEKITLSP_CCOMPLETIONSCORING_H + +#define _GNU_SOURCE +#include + +static inline void *sourcekitlsp_memmem(const void *haystack, size_t haystacklen, const void *needle, size_t needlelen) { + return memmem(haystack, haystacklen, needle, needlelen); +} + +#endif // SOURCEKITLSP_CCOMPLETIONSCORING_H diff --git a/Sources/CompletionScoring/Utilities/SwiftExtensions.swift b/Sources/CompletionScoring/Utilities/SwiftExtensions.swift index fdea664fc..1a71d9701 100644 --- a/Sources/CompletionScoring/Utilities/SwiftExtensions.swift +++ b/Sources/CompletionScoring/Utilities/SwiftExtensions.swift @@ -10,20 +10,9 @@ // //===----------------------------------------------------------------------===// +import CCompletionScoring import Foundation -#if canImport(Darwin) -import Darwin -#elseif canImport(Glibc) -import Glibc -#elseif canImport(Musl) -import Musl -#elseif canImport(CRT) -import CRT -#elseif canImport(Bionic) -import Bionic -#endif - extension Range where Bound: Numeric { init(from: Bound, length: Bound) { self = from..<(from + length) @@ -447,7 +436,9 @@ extension UnsafeBufferPointer { guard needle.count > 0, let needleBaseAddress = needle.baseAddress else { return nil } - guard let match = memmem(baseAddress + startOffset, count - startOffset, needleBaseAddress, needle.count) else { + guard + let match = sourcekitlsp_memmem(baseAddress + startOffset, count - startOffset, needleBaseAddress, needle.count) + else { return nil } let start = baseAddress.distance(to: match.assumingMemoryBound(to: UInt8.self)) From 52e035184439fb25286a20be6e45082c21c574aa Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Wed, 8 Jan 2025 22:42:47 +0100 Subject: [PATCH 15/18] Use `UInt64.random` instead of `arc4random` --- .../RepeatableRandomNumberGenerator.swift | 4 ++-- Sources/CompletionScoringTestSupport/TestExtensions.swift | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Sources/CompletionScoringTestSupport/RepeatableRandomNumberGenerator.swift b/Sources/CompletionScoringTestSupport/RepeatableRandomNumberGenerator.swift index 15d091654..20f2e88f2 100644 --- a/Sources/CompletionScoringTestSupport/RepeatableRandomNumberGenerator.swift +++ b/Sources/CompletionScoringTestSupport/RepeatableRandomNumberGenerator.swift @@ -60,8 +60,8 @@ package struct RepeatableRandomNumberGenerator: RandomNumberGenerator { static func generateSeed() { let numbers: [UInt64] = (0..<1024).map { _ in - let lo = UInt64(arc4random()) - let hi = UInt64(arc4random()) + let lo = UInt64.random(in: 0...UInt64.max) + let hi = UInt64.random(in: 0...UInt64.max) return (hi << 32) | lo } let header = diff --git a/Sources/CompletionScoringTestSupport/TestExtensions.swift b/Sources/CompletionScoringTestSupport/TestExtensions.swift index e49b5ca32..e18ec7f9c 100644 --- a/Sources/CompletionScoringTestSupport/TestExtensions.swift +++ b/Sources/CompletionScoringTestSupport/TestExtensions.swift @@ -47,7 +47,7 @@ extension XCTestCase { while elapsed < 1.0 { elapsed += duration { let integers = Array(count: integers) { - arc4random() + UInt64.random(in: 0...UInt64.max) } DispatchQueue.concurrentPerform(iterations: 128) { _ in integers.withUnsafeBytes { bytes in From 16f897aefa78d1d7c9e4a2fd2c5f730602b1f6d3 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Fri, 10 Jan 2025 10:18:09 -0800 Subject: [PATCH 16/18] Make the SourceKit plugins work on Windows --- Documentation/Environment Variables.md | 1 + Package.swift | 6 ++- Sources/CCompletionScoring/CMakeLists.txt | 2 + .../include/CCompletionScoring.h | 24 ++++++++- .../include/module.modulemap | 4 ++ Sources/CMakeLists.txt | 5 ++ Sources/CompletionScoring/CMakeLists.txt | 51 +++++++++++++++++++ .../include/CodeCompletionSwiftInterop.h | 36 ++++++------- Sources/SKLogging/CMakeLists.txt | 17 ++++++- Sources/SKTestSupport/PluginPaths.swift | 22 +++++--- Sources/SKUtilities/CMakeLists.txt | 19 +++++-- Sources/SourceKitD/CMakeLists.txt | 22 ++++++-- Sources/SwiftExtensions/CMakeLists.txt | 15 ++++-- .../SwiftSourceKitClientPlugin/CMakeLists.txt | 18 +++++++ .../ClientPlugin.swift | 2 +- Sources/SwiftSourceKitPlugin/CMakeLists.txt | 51 +++++++++++++++++++ .../SwiftSourceKitPluginCommon/CMakeLists.txt | 18 +++++++ .../SwiftExtensionsTests.swift | 4 +- .../SwiftSourceKitPluginTests.swift | 4 ++ 19 files changed, 277 insertions(+), 44 deletions(-) create mode 100644 Sources/CCompletionScoring/CMakeLists.txt create mode 100644 Sources/CCompletionScoring/include/module.modulemap create mode 100644 Sources/CompletionScoring/CMakeLists.txt create mode 100644 Sources/SwiftSourceKitClientPlugin/CMakeLists.txt create mode 100644 Sources/SwiftSourceKitPlugin/CMakeLists.txt create mode 100644 Sources/SwiftSourceKitPluginCommon/CMakeLists.txt diff --git a/Documentation/Environment Variables.md b/Documentation/Environment Variables.md index c88ed14d9..53005b759 100644 --- a/Documentation/Environment Variables.md +++ b/Documentation/Environment Variables.md @@ -18,3 +18,4 @@ The following environment variables can be used to control some behavior in Sour - `SOURCEKIT_LSP_KEEP_TEST_SCRATCH_DIR`: Does not delete the temporary files created during test execution. Allows inspection of the test projects after the test finishes. - `SOURCEKIT_LSP_TEST_MODULE_CACHE`: Specifies where tests should store their shared module cache. Defaults to writing the module cache to a temporary directory. Intended so that CI systems can clean the module cache directory after running. - `SOURCEKIT_LSP_TEST_TIMEOUT`: Override the timeout duration for tests, in seconds. +- `SOURCEKIT_LSP_TEST_PLUGIN_PATHS`: Load the SourceKit plugins from this path instead of relative to the package's build folder. diff --git a/Package.swift b/Package.swift index 9889292ed..3769de30a 100644 --- a/Package.swift +++ b/Package.swift @@ -540,11 +540,13 @@ var targets: [Target] = [ dependencies: [ "Csourcekitd", "SourceKitDForPlugin", + "SwiftExtensionsForPlugin", "SwiftSourceKitPluginCommon", ], swiftSettings: globalSwiftSettings + [ .unsafeFlags([ "-module-alias", "SourceKitD=SourceKitDForPlugin", + "-module-alias", "SwiftExtensions=SwiftExtensionsForPlugin", ]) ] ), @@ -662,11 +664,11 @@ var targets: [Target] = [ if buildOnlyTests { products = [] targets = targets.compactMap { target in - guard target.isTest || target.name == "SKTestSupport" else { + guard target.isTest || target.name.contains("TestSupport") else { return nil } target.dependencies = target.dependencies.filter { dependency in - if case .byNameItem(name: "SKTestSupport", _) = dependency { + if case .byNameItem(name: let name, _) = dependency, name.contains("TestSupport") { return true } return false diff --git a/Sources/CCompletionScoring/CMakeLists.txt b/Sources/CCompletionScoring/CMakeLists.txt new file mode 100644 index 000000000..18437046e --- /dev/null +++ b/Sources/CCompletionScoring/CMakeLists.txt @@ -0,0 +1,2 @@ +add_library(CCompletionScoring INTERFACE) +target_include_directories(CCompletionScoring INTERFACE "include") diff --git a/Sources/CCompletionScoring/include/CCompletionScoring.h b/Sources/CCompletionScoring/include/CCompletionScoring.h index 875c55c9f..6f6d60ba0 100644 --- a/Sources/CCompletionScoring/include/CCompletionScoring.h +++ b/Sources/CCompletionScoring/include/CCompletionScoring.h @@ -16,8 +16,28 @@ #define _GNU_SOURCE #include -static inline void *sourcekitlsp_memmem(const void *haystack, size_t haystacklen, const void *needle, size_t needlelen) { - return memmem(haystack, haystacklen, needle, needlelen); +static inline void *sourcekitlsp_memmem(const void *haystack, size_t haystack_len, const void *needle, size_t needle_len) { + #if defined(_WIN32) && !defined(__CYGWIN__) + // memmem is not available on Windows + if (!haystack || haystack_len == 0) { + return NULL; + } + if (!needle || needle_len == 0) { + return NULL; + } + if (needle_len > haystack_len) { + return NULL; + } + + for (size_t offset = 0; offset <= haystack_len - needle_len; ++offset) { + if (memcmp(haystack + offset, needle, needle_len) == 0) { + return (void *)haystack + offset; + } + } + return NULL; + #else + return memmem(haystack, haystack_len, needle, needle_len); + #endif } #endif // SOURCEKITLSP_CCOMPLETIONSCORING_H diff --git a/Sources/CCompletionScoring/include/module.modulemap b/Sources/CCompletionScoring/include/module.modulemap new file mode 100644 index 000000000..533a86eb3 --- /dev/null +++ b/Sources/CCompletionScoring/include/module.modulemap @@ -0,0 +1,4 @@ +module CCompletionScoring { + header "CCompletionScoring.h" + export * +} diff --git a/Sources/CMakeLists.txt b/Sources/CMakeLists.txt index c32f4d8db..926ea6265 100644 --- a/Sources/CMakeLists.txt +++ b/Sources/CMakeLists.txt @@ -4,6 +4,8 @@ add_compile_options("$<$:SHELL:-swift-version 6>") add_subdirectory(BuildServerProtocol) add_subdirectory(BuildSystemIntegration) add_subdirectory(CAtomics) +add_subdirectory(CCompletionScoring) +add_subdirectory(CompletionScoring) add_subdirectory(Csourcekitd) add_subdirectory(Diagnose) add_subdirectory(InProcessClient) @@ -18,5 +20,8 @@ add_subdirectory(SourceKitLSP) add_subdirectory(SourceKitD) add_subdirectory(sourcekit-lsp) add_subdirectory(SwiftExtensions) +add_subdirectory(SwiftSourceKitClientPlugin) +add_subdirectory(SwiftSourceKitPlugin) +add_subdirectory(SwiftSourceKitPluginCommon) add_subdirectory(ToolchainRegistry) add_subdirectory(TSCExtensions) diff --git a/Sources/CompletionScoring/CMakeLists.txt b/Sources/CompletionScoring/CMakeLists.txt new file mode 100644 index 000000000..1356e009b --- /dev/null +++ b/Sources/CompletionScoring/CMakeLists.txt @@ -0,0 +1,51 @@ +set(sources + Semantics/SemanticClassification.swift + Semantics/CompletionScore.swift + Semantics/Components + Semantics/Components/TypeCompatibility.swift + Semantics/Components/StructuralProximity.swift + Semantics/Components/Flair.swift + Semantics/Components/CompletionKind.swift + Semantics/Components/Popularity.swift + Semantics/Components/ScopeProximity.swift + Semantics/Components/Availability.swift + Semantics/Components/SynchronicityCompatibility.swift + Semantics/Components/PopularityIndex.swift + Semantics/Components/ModuleProximity.swift + Semantics/Components/PopularityTable.swift + Utilities/UnsafeStackAllocator.swift + Utilities/Serialization + Utilities/Serialization/BinaryCodable.swift + Utilities/Serialization/BinaryEncoder.swift + Utilities/Serialization/BinaryDecoder.swift + Utilities/Serialization/Conformances + Utilities/Serialization/Conformances/Dictionary+BinaryCodable.swift + Utilities/Serialization/Conformances/OptionSet+BinaryCodable.swift + Utilities/Serialization/Conformances/Optional+BinaryCodable.swift + Utilities/Serialization/Conformances/Scalars+BinaryCodable.swift + Utilities/Serialization/Conformances/String+BinaryCodable.swift + Utilities/Serialization/Conformances/Array+BinaryCodable.swift + Utilities/SwiftExtensions.swift + Utilities/SelectTopK.swift + Utilities/UnsafeArray.swift + Text/CandidateBatch.swift + Text/MatchCollator.Match.swift + Text/UTF8Byte.swift + Text/MatchCollator.swift + Text/InfluencingIdentifiers.swift + Text/MatchCollator.Selection.swift + Text/ScoredMatchSelector.swift + Text/RejectionFilter.swift + Text/Pattern.swift) + +add_library(CompletionScoring STATIC ${sources}) +set_target_properties(CompletionScoring PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_Swift_MODULE_DIRECTORY}) +target_link_libraries(CompletionScoring PUBLIC + CCompletionScoring) + +add_library(CompletionScoringForPlugin STATIC ${sources}) +set_target_properties(CompletionScoringForPlugin PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_Swift_MODULE_DIRECTORY}) +target_link_libraries(CompletionScoringForPlugin PUBLIC + CCompletionScoring) diff --git a/Sources/Csourcekitd/include/CodeCompletionSwiftInterop.h b/Sources/Csourcekitd/include/CodeCompletionSwiftInterop.h index 8492145ec..7d0955e7e 100644 --- a/Sources/Csourcekitd/include/CodeCompletionSwiftInterop.h +++ b/Sources/Csourcekitd/include/CodeCompletionSwiftInterop.h @@ -24,7 +24,7 @@ typedef void *swiftide_api_connection_t; /// may be more expensive to compute. typedef void *swiftide_api_completion_item_t; -typedef enum { +enum swiftide_api_completion_kind_t: uint32_t { SWIFTIDE_COMPLETION_KIND_NONE = 0, SWIFTIDE_COMPLETION_KIND_IMPORT = 1, SWIFTIDE_COMPLETION_KIND_UNRESOLVEDMEMBER = 2, @@ -69,17 +69,17 @@ typedef enum { SWIFTIDE_COMPLETION_KIND_TYPESIMPLEORCOMPOSITION = 41, SWIFTIDE_COMPLETION_KIND_TYPEPOSSIBLEFUNCTIONPARAMBEGINNING = 42, SWIFTIDE_COMPLETION_KIND_TYPEATTRINHERITANCEBEGINNING = 43, -} swiftide_api_completion_kind_t; +}; -typedef enum { +enum swiftide_api_completion_item_kind_t: uint32_t { SWIFTIDE_COMPLETION_ITEM_KIND_DECLARATION = 0, SWIFTIDE_COMPLETION_ITEM_KIND_KEYWORD = 1, SWIFTIDE_COMPLETION_ITEM_KIND_PATTERN = 2, SWIFTIDE_COMPLETION_ITEM_KIND_LITERAL = 3, SWIFTIDE_COMPLETION_ITEM_KIND_BUILTINOPERATOR = 4, -} swiftide_api_completion_item_kind_t; +}; -typedef enum { +enum swiftide_api_completion_item_decl_kind_t: uint32_t { SWIFTIDE_COMPLETION_ITEM_DECL_KIND_MODULE = 0, SWIFTIDE_COMPLETION_ITEM_DECL_KIND_CLASS = 1, SWIFTIDE_COMPLETION_ITEM_DECL_KIND_STRUCT = 2, @@ -105,18 +105,18 @@ typedef enum { SWIFTIDE_COMPLETION_ITEM_DECL_KIND_PRECEDENCEGROUP = 22, SWIFTIDE_COMPLETION_ITEM_DECL_KIND_ACTOR = 23, SWIFTIDE_COMPLETION_ITEM_DECL_KIND_MACRO = 24, -} swiftide_api_completion_item_decl_kind_t; +}; -typedef enum { +enum swiftide_api_completion_type_relation_t: uint32_t { SWIFTIDE_COMPLETION_TYPE_RELATION_NOTAPPLICABLE = 0, SWIFTIDE_COMPLETION_TYPE_RELATION_UNKNOWN = 1, SWIFTIDE_COMPLETION_TYPE_RELATION_UNRELATED = 2, SWIFTIDE_COMPLETION_TYPE_RELATION_INVALID = 3, SWIFTIDE_COMPLETION_TYPE_RELATION_CONVERTIBLE = 4, SWIFTIDE_COMPLETION_TYPE_RELATION_IDENTICAL = 5, -} swiftide_api_completion_type_relation_t; +}; -typedef enum { +enum swiftide_api_completion_semantic_context_t: uint32_t { SWIFTIDE_COMPLETION_SEMANTIC_CONTEXT_NONE = 0, /* obsoleted */SWIFTIDE_COMPLETION_SEMANTIC_CONTEXT_EXPRESSIONSPECIFIC = 1, SWIFTIDE_COMPLETION_SEMANTIC_CONTEXT_LOCAL = 2, @@ -125,9 +125,9 @@ typedef enum { SWIFTIDE_COMPLETION_SEMANTIC_CONTEXT_OUTSIDENOMINAL = 5, SWIFTIDE_COMPLETION_SEMANTIC_CONTEXT_CURRENTMODULE = 6, SWIFTIDE_COMPLETION_SEMANTIC_CONTEXT_OTHERMODULE = 7, -} swiftide_api_completion_semantic_context_t; +}; -typedef enum { +enum swiftide_api_completion_flair_t: uint32_t { SWIFTIDE_COMPLETION_FLAIR_EXPRESSIONSPECIFIC = 1 << 0, SWIFTIDE_COMPLETION_FLAIR_SUPERCHAIN = 1 << 1, SWIFTIDE_COMPLETION_FLAIR_ARGUMENTLABELS = 1 << 2, @@ -135,9 +135,9 @@ typedef enum { SWIFTIDE_COMPLETION_FLAIR_RAREKEYWORDATCURRENTPOSITION = 1 << 4, SWIFTIDE_COMPLETION_FLAIR_RARETYPEATCURRENTPOSITION = 1 << 5, SWIFTIDE_COMPLETION_FLAIR_EXPRESSIONATNONSCRIPTORMAINFILESCOPE = 1 << 6, -} swiftide_api_completion_flair_t; +}; -typedef enum { +enum swiftide_api_completion_not_recommended_reason_t: uint32_t { SWIFTIDE_COMPLETION_NOT_RECOMMENDED_NONE = 0, SWIFTIDE_COMPLETION_NOT_RECOMMENDED_REDUNDANT_IMPORT = 1, SWIFTIDE_COMPLETION_NOT_RECOMMENDED_DEPRECATED = 2, @@ -147,15 +147,15 @@ typedef enum { SWIFTIDE_COMPLETION_NOT_RECOMMENDED_REDUNDANT_IMPORT_INDIRECT = 6, SWIFTIDE_COMPLETION_NOT_RECOMMENDED_SOFTDEPRECATED = 7, SWIFTIDE_COMPLETION_NOT_RECOMMENDED_NON_ASYNC_ALTERNATIVE_USED_IN_ASYNC_CONTEXT = 8, -} swiftide_api_completion_not_recommended_reason_t; +}; -typedef enum { +enum swiftide_api_completion_diagnostic_severity_t: uint32_t { SWIFTIDE_COMPLETION_DIAGNOSTIC_SEVERITY_NONE = 0, SWIFTIDE_COMPLETION_DIAGNOSTIC_SEVERITY_ERROR = 1, SWIFTIDE_COMPLETION_DIAGNOSTIC_SEVERITY_WARNING = 2, SWIFTIDE_COMPLETION_DIAGNOSTIC_SEVERITY_REMARK = 3, SWIFTIDE_COMPLETION_DIAGNOSTIC_SEVERITY_NOTE = 4, -} swiftide_api_completion_diagnostic_severity_t; +}; typedef void *swiftide_api_completion_request_t; @@ -278,7 +278,7 @@ typedef struct { uint64_t index ); - swiftide_api_completion_kind_t (*_Nonnull completion_result_get_kind)( + enum swiftide_api_completion_kind_t (*_Nonnull completion_result_get_kind)( _Null_unspecified swiftide_api_completion_response_t ); @@ -361,7 +361,7 @@ typedef struct { void (*_Nonnull completion_item_get_diagnostic)( _Null_unspecified swiftide_api_completion_response_t, _Null_unspecified swiftide_api_completion_item_t, - void (^_Null_unspecified handler)(swiftide_api_completion_diagnostic_severity_t, const char *_Null_unspecified) + void (^_Null_unspecified handler)(enum swiftide_api_completion_diagnostic_severity_t, const char *_Null_unspecified) ); bool (*_Nonnull completion_item_is_system)( diff --git a/Sources/SKLogging/CMakeLists.txt b/Sources/SKLogging/CMakeLists.txt index 5f3d26e45..1c9b6a9f6 100644 --- a/Sources/SKLogging/CMakeLists.txt +++ b/Sources/SKLogging/CMakeLists.txt @@ -1,4 +1,4 @@ -add_library(SKLogging STATIC +set(sources CustomLogStringConvertible.swift Error+ForLogging.swift Logging.swift @@ -7,6 +7,8 @@ add_library(SKLogging STATIC OrLog.swift SetGlobalLogFileHandler.swift SplitLogMessage.swift) + +add_library(SKLogging STATIC ${sources}) set_target_properties(SKLogging PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_Swift_MODULE_DIRECTORY}) target_link_libraries(SKLogging PRIVATE @@ -14,3 +16,16 @@ target_link_libraries(SKLogging PRIVATE target_link_libraries(SKLogging PUBLIC SwiftExtensions Crypto) + +add_library(SKLoggingForPlugin STATIC ${sources}) +set_target_properties(SKLoggingForPlugin PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_Swift_MODULE_DIRECTORY}) +target_compile_options(SKLoggingForPlugin PRIVATE + $<$: + -DNO_CRYPTO_DEPENDENCY; + "SHELL:-module-alias SwiftExtensions=SwiftExtensionsForPlugin" + >) +target_link_libraries(SKLoggingForPlugin PRIVATE + $<$>:Foundation>) +target_link_libraries(SKLoggingForPlugin PUBLIC + SwiftExtensionsForPlugin) diff --git a/Sources/SKTestSupport/PluginPaths.swift b/Sources/SKTestSupport/PluginPaths.swift index 88a0dd494..4ef47d3e3 100644 --- a/Sources/SKTestSupport/PluginPaths.swift +++ b/Sources/SKTestSupport/PluginPaths.swift @@ -88,14 +88,17 @@ private func pluginPaths(relativeTo base: URL) -> PluginPaths? { // When building using 'swift test' do { #if canImport(Darwin) - let dylibExtension = "dylib" + let clientPluginName = "libSwiftSourceKitClientPlugin.dylib" + let servicePluginName = "libSwiftSourceKitPlugin.dylib" #elseif os(Windows) - let dylibExtension = "dll" + let clientPluginName = "SwiftSourceKitClientPlugin.dll" + let servicePluginName = "SwiftSourceKitPlugin.dll" #else - let dylibExtension = "so" + let clientPluginName = "libSwiftSourceKitClientPlugin.so" + let servicePluginName = "libSwiftSourceKitPlugin.so" #endif - let clientPlugin = base.appendingPathComponent("libSwiftSourceKitClientPlugin.\(dylibExtension)") - let servicePlugin = base.appendingPathComponent("libSwiftSourceKitPlugin.\(dylibExtension)") + let clientPlugin = base.appendingPathComponent(clientPluginName) + let servicePlugin = base.appendingPathComponent(servicePluginName) if fileExists(at: clientPlugin) && fileExists(at: servicePlugin) { return PluginPaths(clientPlugin: clientPlugin, servicePlugin: servicePlugin) } @@ -113,7 +116,12 @@ package var sourceKitPluginPaths: PluginPaths { "Could not find SourceKit plugin. Ensure that you build the entire SourceKit-LSP package before running tests." } - var base = xctestBundle + var base = + if let pluginPaths = ProcessInfo.processInfo.environment["SOURCEKIT_LSP_TEST_PLUGIN_PATHS"] { + URL(fileURLWithPath: pluginPaths) + } else { + xctestBundle + } while base.pathComponents.count > 1 { if let paths = pluginPaths(relativeTo: base) { return paths @@ -121,8 +129,6 @@ package var sourceKitPluginPaths: PluginPaths { base = base.deletingLastPathComponent() } - // If we couldn't find the plugins, keep `didLoadPlugin = false`, which will throw an error in each test case's - // `setUp` function. We can't throw an error from the class `setUp` function. throw PluginLoadingError() } } diff --git a/Sources/SKUtilities/CMakeLists.txt b/Sources/SKUtilities/CMakeLists.txt index 791eb8870..9d29ca202 100644 --- a/Sources/SKUtilities/CMakeLists.txt +++ b/Sources/SKUtilities/CMakeLists.txt @@ -1,13 +1,26 @@ - -add_library(SKUtilities STATIC +set(sources Debouncer.swift Dictionary+InitWithElementsKeyedBy.swift LineTable.swift ) + +add_library(SKUtilities STATIC ${sources}) set_target_properties(SKUtilities PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_Swift_MODULE_DIRECTORY}) target_link_libraries(SKUtilities PRIVATE SKLogging SwiftExtensions - TSCBasic $<$>:Foundation>) + +add_library(SKUtilitiesForPlugin STATIC ${sources}) +set_target_properties(SKUtilitiesForPlugin PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_Swift_MODULE_DIRECTORY}) +target_compile_options(SKUtilitiesForPlugin PRIVATE + $<$: + "SHELL:-module-alias SKLogging=SKLoggingForPlugin" + "SHELL:-module-alias SwiftExtensions=SwiftExtensionsForPlugin" + >) +target_link_libraries(SKUtilitiesForPlugin PRIVATE + SKLoggingForPlugin + SwiftExtensionsForPlugin + $<$>:Foundation>) \ No newline at end of file diff --git a/Sources/SourceKitD/CMakeLists.txt b/Sources/SourceKitD/CMakeLists.txt index 2a8515ca2..3a261e449 100644 --- a/Sources/SourceKitD/CMakeLists.txt +++ b/Sources/SourceKitD/CMakeLists.txt @@ -1,5 +1,4 @@ - -add_library(SourceKitD STATIC +set(sources dlopen.swift SKDRequestArray.swift SKDRequestDictionary.swift @@ -11,6 +10,8 @@ add_library(SourceKitD STATIC SourceKitDRegistry.swift sourcekitd_functions.swift sourcekitd_uids.swift) + +add_library(SourceKitD STATIC ${sources}) set_target_properties(SourceKitD PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_Swift_MODULE_DIRECTORY}) target_link_libraries(SourceKitD PUBLIC @@ -18,5 +19,20 @@ target_link_libraries(SourceKitD PUBLIC target_link_libraries(SourceKitD PRIVATE SKLogging SwiftExtensions - TSCBasic + $<$>:Foundation>) + + +add_library(SourceKitDForPlugin STATIC ${sources}) +set_target_properties(SourceKitDForPlugin PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_Swift_MODULE_DIRECTORY}) +target_compile_options(SourceKitDForPlugin PRIVATE + $<$: + "SHELL:-module-alias SKLogging=SKLoggingForPlugin" + "SHELL:-module-alias SwiftExtensions=SwiftExtensionsForPlugin" + >) +target_link_libraries(SourceKitDForPlugin PUBLIC + Csourcekitd) +target_link_libraries(SourceKitDForPlugin PRIVATE + SKLoggingForPlugin + SwiftExtensionsForPlugin $<$>:Foundation>) diff --git a/Sources/SwiftExtensions/CMakeLists.txt b/Sources/SwiftExtensions/CMakeLists.txt index 53064e8ff..f300ac48e 100644 --- a/Sources/SwiftExtensions/CMakeLists.txt +++ b/Sources/SwiftExtensions/CMakeLists.txt @@ -1,5 +1,4 @@ - -add_library(SwiftExtensions STATIC +set(sources Array+Safe.swift AsyncQueue.swift AsyncUtils.swift @@ -20,11 +19,19 @@ add_library(SwiftExtensions STATIC TransitiveClosure.swift URLExtensions.swift ) + +add_library(SwiftExtensions STATIC ${sources}) set_target_properties(SwiftExtensions PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_Swift_MODULE_DIRECTORY}) - target_link_libraries(SwiftExtensions PUBLIC CAtomics) - target_link_libraries(SwiftExtensions PRIVATE $<$>:Foundation>) + +add_library(SwiftExtensionsForPlugin STATIC ${sources}) +set_target_properties(SwiftExtensionsForPlugin PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_Swift_MODULE_DIRECTORY}) +target_link_libraries(SwiftExtensionsForPlugin PUBLIC + CAtomics) +target_link_libraries(SwiftExtensionsForPlugin PRIVATE + $<$>:Foundation>) diff --git a/Sources/SwiftSourceKitClientPlugin/CMakeLists.txt b/Sources/SwiftSourceKitClientPlugin/CMakeLists.txt new file mode 100644 index 000000000..9a2b1f5a5 --- /dev/null +++ b/Sources/SwiftSourceKitClientPlugin/CMakeLists.txt @@ -0,0 +1,18 @@ +add_library(SwiftSourceKitClientPlugin SHARED + ClientPlugin.swift) + +set_target_properties(SwiftSourceKitClientPlugin PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_Swift_MODULE_DIRECTORY}) +target_compile_options(SwiftSourceKitClientPlugin PRIVATE + $<$: + "SHELL:-module-alias SourceKitD=SourceKitDForPlugin" + "SHELL:-module-alias SwiftExtensions=SwiftExtensionsForPlugin" + >) +target_link_libraries(SwiftSourceKitClientPlugin PRIVATE + Csourcekitd + SourceKitD + SwiftExtensions + SwiftSourceKitPluginCommon + $<$>:FoundationXML>) + +install(TARGETS SwiftSourceKitClientPlugin DESTINATION lib) diff --git a/Sources/SwiftSourceKitClientPlugin/ClientPlugin.swift b/Sources/SwiftSourceKitClientPlugin/ClientPlugin.swift index c326ff5c5..d035bb5b5 100644 --- a/Sources/SwiftSourceKitClientPlugin/ClientPlugin.swift +++ b/Sources/SwiftSourceKitClientPlugin/ClientPlugin.swift @@ -12,7 +12,7 @@ import Foundation import SourceKitD -import SwiftExtensionsForPlugin +import SwiftExtensions import SwiftSourceKitPluginCommon #if compiler(>=6) diff --git a/Sources/SwiftSourceKitPlugin/CMakeLists.txt b/Sources/SwiftSourceKitPlugin/CMakeLists.txt new file mode 100644 index 000000000..342c11bd2 --- /dev/null +++ b/Sources/SwiftSourceKitPlugin/CMakeLists.txt @@ -0,0 +1,51 @@ +add_library(SwiftSourceKitPlugin SHARED + ASTCompletion/ASTCompletionItem.swift + ASTCompletion/CompletionSession.swift + SKDResponse.swift + SKDResponseArrayBuilder.swift + SourceKitPluginError.swift + SKDRequestDictionaryReader.swift + SKDRequestArrayReader.swift + CompletionResultsArrayBuilder.swift + SKDResponseValue.swift + CompletionProvider.swift + SKDResponseDictionaryBuilder.swift + CodeCompletion/Location.swift + CodeCompletion/Connection.swift + CodeCompletion/CompletionContext.swift + CodeCompletion/Completion+ItemKind.swift + CodeCompletion/Completion+SemanticContext.swift + CodeCompletion/Completion+Identifier.swift + CodeCompletion/TextEdit.swift + CodeCompletion/Completion+TypeRelation.swift + CodeCompletion/CompletionItem.swift + CodeCompletion/WithArrayOfCStrings.swift + CodeCompletion/CompletionOptions.swift + CodeCompletion/Completion+Diagnostic.swift + CodeCompletion/CompletionSorting.swift + CodeCompletion/Completion+PriorityBucket.swift + CodeCompletion/Position.swift + SourceKitDWrappers.swift + Plugin.swift) + +set_target_properties(SwiftSourceKitPlugin PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_Swift_MODULE_DIRECTORY}) +target_compile_options(SwiftSourceKitPlugin PRIVATE + $<$: + "SHELL:-module-alias CompletionScoring=CompletionScoringForPlugin" + "SHELL:-module-alias SKUtilities=SKUtilitiesForPlugin" + "SHELL:-module-alias SourceKitD=SourceKitDForPlugin" + "SHELL:-module-alias SKLogging=SKLoggingForPlugin" + "SHELL:-module-alias SwiftExtensions=SwiftExtensionsForPlugin" + >) +target_link_libraries(SwiftSourceKitPlugin PRIVATE + Csourcekitd + CompletionScoringForPlugin + SKUtilitiesForPlugin + SKLoggingForPlugin + SourceKitDForPlugin + SwiftSourceKitPluginCommon + SwiftExtensionsForPlugin + $<$>:FoundationXML>) + +install(TARGETS SwiftSourceKitPlugin DESTINATION lib) diff --git a/Sources/SwiftSourceKitPluginCommon/CMakeLists.txt b/Sources/SwiftSourceKitPluginCommon/CMakeLists.txt new file mode 100644 index 000000000..747ef4a2a --- /dev/null +++ b/Sources/SwiftSourceKitPluginCommon/CMakeLists.txt @@ -0,0 +1,18 @@ +add_library(SwiftSourceKitPluginCommon STATIC + CompletionResultsArray.swift + DynamicallyLoadedSourceKitdD+forPlugin.swift) + +target_compile_options(SwiftSourceKitPluginCommon PRIVATE + $<$: + "SHELL:-module-alias SourceKitD=SourceKitDForPlugin" + "SHELL:-module-alias SKLogging=SKLoggingForPlugin" + "SHELL:-module-alias SwiftExtensions=SwiftExtensionsForPlugin" + >) +set_target_properties(SwiftSourceKitPluginCommon PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_Swift_MODULE_DIRECTORY}) +target_link_libraries(SwiftSourceKitPluginCommon PRIVATE + Csourcekitd + SourceKitDForPlugin + SKLoggingForPlugin + SwiftExtensionsForPlugin + $<$>:FoundationXML>) \ No newline at end of file diff --git a/Tests/CompletionScoringTests/SwiftExtensionsTests.swift b/Tests/CompletionScoringTests/SwiftExtensionsTests.swift index 233d0780a..d0c86f7a0 100644 --- a/Tests/CompletionScoringTests/SwiftExtensionsTests.swift +++ b/Tests/CompletionScoringTests/SwiftExtensionsTests.swift @@ -103,11 +103,11 @@ class SwiftExtensionsTests: XCTestCase { } func testRangeOfBytes() { - func search(for needle: String, in body: String, expecting expectedMatch: Range?) { + func search(for needle: String, in body: String, expecting expectedMatch: Range?, line: UInt = #line) { needle.withUncachedUTF8Bytes { needle in body.withUncachedUTF8Bytes { body in let actualMatch = body.rangeOf(bytes: needle) - XCTAssertEqual(actualMatch, expectedMatch) + XCTAssertEqual(actualMatch, expectedMatch, line: line) } } } diff --git a/Tests/SwiftSourceKitPluginTests/SwiftSourceKitPluginTests.swift b/Tests/SwiftSourceKitPluginTests/SwiftSourceKitPluginTests.swift index 38b4ec900..edccc0eb4 100644 --- a/Tests/SwiftSourceKitPluginTests/SwiftSourceKitPluginTests.swift +++ b/Tests/SwiftSourceKitPluginTests/SwiftSourceKitPluginTests.swift @@ -24,7 +24,11 @@ final class SwiftSourceKitPluginTests: XCTestCase { /// /// The file does not actually exist on disk. private func scratchFilePath(testName: String = #function, fileName: String = "a.swift") -> String { + #if os(Windows) + return "C:\\\(testScratchName(testName: testName))\\\(fileName)" + #else return "/\(testScratchName(testName: testName))/\(fileName)" + #endif } func getSourceKitD() async throws -> SourceKitD { From 58f7766aa15b211a3b3ffe0624593dd3abf2f5f5 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Mon, 13 Jan 2025 08:56:46 -0800 Subject: [PATCH 17/18] Install SourceKit plugin to `lib` instead of `bin` --- Utilities/build-script-helper.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Utilities/build-script-helper.py b/Utilities/build-script-helper.py index 9f7fc615c..472437745 100755 --- a/Utilities/build-script-helper.py +++ b/Utilities/build-script-helper.py @@ -254,10 +254,15 @@ def install(swift_exec: str, args: argparse.Namespace) -> None: build_single_product('SwiftSourceKitPlugin', swift_exec, args) build_single_product('SwiftSourceKitClientPlugin', swift_exec, args) + if platform.system() == 'Darwin': + dynamic_library_extension = "dylib" + else: + dynamic_library_extension = "so" + for prefix in args.install_prefixes: install_binary('sourcekit-lsp', bin_path, os.path.join(prefix, 'bin'), verbose=args.verbose) - install_binary('libSwiftSourceKitPlugin.dylib', bin_path, os.path.join(prefix, 'bin'), verbose=args.verbose) - install_binary('libSwiftSourceKitClientPlugin.dylib', bin_path, os.path.join(prefix, 'bin'), verbose=args.verbose) + install_binary(f'libSwiftSourceKitPlugin.{dynamic_library_extension}', bin_path, os.path.join(prefix, 'lib'), verbose=args.verbose) + install_binary(f'libSwiftSourceKitClientPlugin.{dynamic_library_extension}', bin_path, os.path.join(prefix, 'lib'), verbose=args.verbose) def handle_invocation(swift_exec: str, args: argparse.Namespace) -> None: From 43a81ae23266a0a2f97e443eaae1f4182ba28d93 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Mon, 13 Jan 2025 14:10:11 -0800 Subject: [PATCH 18/18] =?UTF-8?q?Don=E2=80=99t=20include=20local=20path=20?= =?UTF-8?q?in=20SourceKit=20plugin?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Package.swift | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Package.swift b/Package.swift index 3769de30a..c7f6e2300 100644 --- a/Package.swift +++ b/Package.swift @@ -548,7 +548,8 @@ var targets: [Target] = [ "-module-alias", "SourceKitD=SourceKitDForPlugin", "-module-alias", "SwiftExtensions=SwiftExtensionsForPlugin", ]) - ] + ], + linkerSettings: sourcekitLSPLinkSettings ), // MARK: SwiftSourceKitPluginCommon @@ -591,7 +592,8 @@ var targets: [Target] = [ "-module-alias", "SKLogging=SKLoggingForPlugin", "-module-alias", "SwiftExtensions=SwiftExtensionsForPlugin", ]) - ] + ], + linkerSettings: sourcekitLSPLinkSettings ), .testTarget(