Skip to content

refactor: remove dead code from autocomplete directory #4522

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
markijbema wants to merge 1 commit into
base: main
Choose a base branch
from
+15 −307
Draft

refactor: remove dead code from autocomplete directory #4522

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
125 changes: 0 additions & 125 deletions src/services/continuedev/core/autocomplete/context/ranking/index.test.ts
View file
Open in desktop

This file was deleted.

131 changes: 0 additions & 131 deletions src/services/continuedev/core/autocomplete/context/ranking/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,3 @@
import { RangeInFileWithContents } from "../../../"
import { countTokens } from "../../../llm/countTokens"
import { AutocompleteSnippetDeprecated } from "../../types"
import { HelperVars } from "../../util/HelperVars"

const rx = /[\s.,/#!$%^&*;:{}=\-_`~()[\]]/g
export function getSymbolsForSnippet(snippet: string): Set<string> {
const symbols = snippet
Expand All @@ -11,129 +6,3 @@ export function getSymbolsForSnippet(snippet: string): Set<string> {
.filter((s) => s !== "")
return new Set(symbols)
}

/**
* Calculate similarity as number of shared symbols divided by total number of unique symbols between both.
*/
function jaccardSimilarity(a: string, b: string): number {
const aSet = getSymbolsForSnippet(a)
const bSet = getSymbolsForSnippet(b)
const union = new Set([...aSet, ...bSet]).size

// Avoid division by zero
if (union === 0) {
return 0
}

let intersection = 0
for (const symbol of aSet) {
if (bSet.has(symbol)) {
intersection++
}
}

return intersection / union
}

/**
* Rank code snippets to be used in tab-autocomplete prompt. Returns a sorted version of the snippet array.
*/
export function rankAndOrderSnippets( //MINIMAL_REPO - this isn't actually used in continue
ranges: AutocompleteSnippetDeprecated[],
helper: HelperVars,
): Required<AutocompleteSnippetDeprecated>[] {
const windowAroundCursor =
helper.fullPrefix.slice(-helper.options.slidingWindowSize * helper.options.slidingWindowPrefixPercentage) +
helper.fullSuffix.slice(helper.options.slidingWindowSize * (1 - helper.options.slidingWindowPrefixPercentage))

const snippets: Required<AutocompleteSnippetDeprecated>[] = ranges.map((snippet) => ({
score: snippet.score ?? jaccardSimilarity(snippet.contents, windowAroundCursor),
...snippet,
}))
const uniqueSnippets = deduplicateSnippets(snippets)
return uniqueSnippets.sort((a, b) => a.score - b.score)
}

/**
* Deduplicate code snippets by merging overlapping ranges into a single range.
*/
function deduplicateSnippets(
snippets: Required<AutocompleteSnippetDeprecated>[],
): Required<AutocompleteSnippetDeprecated>[] {
// Group by file
const fileGroups: {
[key: string]: Required<AutocompleteSnippetDeprecated>[]
} = {}
for (const snippet of snippets) {
if (!fileGroups[snippet.filepath]) {
fileGroups[snippet.filepath] = []
}
fileGroups[snippet.filepath].push(snippet)
}

// Merge overlapping ranges
const allRanges = []
for (const file of Object.keys(fileGroups)) {
allRanges.push(...mergeSnippetsByRange(fileGroups[file]))
}
return allRanges
}

function mergeSnippetsByRange(
snippets: Required<AutocompleteSnippetDeprecated>[],
): Required<AutocompleteSnippetDeprecated>[] {
if (snippets.length <= 1) {
return snippets
}

const sorted = snippets.sort((a, b) => a.range.start.line - b.range.start.line)
const merged: Required<AutocompleteSnippetDeprecated>[] = []

while (sorted.length > 0) {
const next = sorted.shift()!
const last = merged[merged.length - 1]
if (merged.length > 0 && last.range.end.line >= next.range.start.line) {
// Merge with previous snippet
last.score = Math.max(last.score, next.score)
try {
last.range.end = next.range.end
} catch (e) {
console.log("Error merging ranges", e)
}
last.contents = mergeOverlappingRangeContents(last, next)
} else {
merged.push(next)
}
}

return merged
}

function mergeOverlappingRangeContents(first: RangeInFileWithContents, second: RangeInFileWithContents): string {
const firstLines = first.contents.split("\n")
const numOverlapping = first.range.end.line - second.range.start.line
return `${firstLines.slice(-numOverlapping).join("\n")}\n${second.contents}`
}

/**
* Fill the allowed space with snippets.
* It is assumed that the snippets are sorted by score.
*/
export function fillPromptWithSnippets( //MINIMAL_REPO - this isn't actually used in continue
snippets: Required<AutocompleteSnippetDeprecated>[],
maxSnippetTokens: number,
modelName: string,
): Required<AutocompleteSnippetDeprecated>[] {
let tokensRemaining = maxSnippetTokens
const keptSnippets: Required<AutocompleteSnippetDeprecated>[] = []
for (let i = 0; i < snippets.length; i++) {
const snippet = snippets[i]
const tokenCount = countTokens(snippet.contents, modelName)
if (tokensRemaining - tokenCount >= 0) {
tokensRemaining -= tokenCount
keptSnippets.push(snippet)
}
}

return keptSnippets
}
55 changes: 14 additions & 41 deletions ...ervices/continuedev/core/autocomplete/context/root-path-context/RootPathContextService.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,27 +6,11 @@ import { Node as SyntaxNode, Query, Point } from "web-tree-sitter"
import { IDE } from "../../.."
import { getFullLanguageName, getQueryForFile, IGNORE_PATH_PATTERNS, LanguageName } from "../../../util/treeSitter"
import { AutocompleteCodeSnippet, AutocompleteSnippetType } from "../../snippets/types"
import { AutocompleteSnippetDeprecated } from "../../types"
import { AstPath } from "../../util/ast"
import { ImportDefinitionsService } from "../ImportDefinitionsService"

// function getSyntaxTreeString(
// node: Parser.SyntaxNode,
// indent: string = "",
// ): string {
// let result = "";
// const nodeInfo = `${node.type} [${node.startPosition.row}:${node.startPosition.column} - ${node.endPosition.row}:${node.endPosition.column}]`;
// result += `${indent}${nodeInfo}\n`;

// for (const child of node.children) {
// result += getSyntaxTreeString(child, indent + " ");
// }

// return result;
// }

export class RootPathContextService {
private cache = new LRUCache<string, AutocompleteSnippetDeprecated[]>({
private cache = new LRUCache<string, AutocompleteCodeSnippet[]>({
max: 100,
})

Expand Down Expand Up @@ -62,8 +46,8 @@ export class RootPathContextService {
.digest("hex")
}

private async getSnippetsForNode(filepath: string, node: SyntaxNode): Promise<AutocompleteSnippetDeprecated[]> {
const snippets: AutocompleteSnippetDeprecated[] = []
private async getSnippetsForNode(filepath: string, node: SyntaxNode): Promise<AutocompleteCodeSnippet[]> {
const snippets: AutocompleteCodeSnippet[] = []
const language = getFullLanguageName(filepath)

let query: Query | undefined
Expand All @@ -72,9 +56,6 @@ export class RootPathContextService {
this.importDefinitionsService.get(filepath)
break
default:
// const type = node.type;
// console.log(getSyntaxTreeString(node));

query = await getQueryForFile(filepath, `root-path-context-queries/${language}/${node.type}.scm`)
break
}
Expand All @@ -100,15 +81,15 @@ export class RootPathContextService {
filepath: string,
endPosition: Point,
language: LanguageName,
): Promise<AutocompleteSnippetDeprecated[]> {
): Promise<AutocompleteCodeSnippet[]> {
const definitions = await this.ide.gotoDefinition({
filepath,
position: {
line: endPosition.row,
character: endPosition.column,
},
})
const newSnippets = await Promise.all(
const newSnippets: AutocompleteCodeSnippet[] = await Promise.all(
definitions
.filter((definition) => {
const isIgnoredPath = IGNORE_PATH_PATTERNS[language]?.some((pattern) =>
Expand All @@ -117,37 +98,29 @@ export class RootPathContextService {

return !isIgnoredPath
})
.map(async (def) => ({
...def,
contents: await this.ide.readRangeInFile(def.filepath, def.range),
})),
.map(
async (def): Promise<AutocompleteCodeSnippet> => ({
filepath: def.filepath,
content: await this.ide.readRangeInFile(def.filepath, def.range),
type: AutocompleteSnippetType.Code,
}),
),
)

return newSnippets
}

async getContextForPath(
filepath: string,
astPath: AstPath,
// cursorIndex: number,
): Promise<AutocompleteCodeSnippet[]> {
async getContextForPath(filepath: string, astPath: AstPath): Promise<AutocompleteCodeSnippet[]> {
const snippets: AutocompleteCodeSnippet[] = []

let parentKey = filepath
for (const astNode of astPath.filter((node) => RootPathContextService.TYPES_TO_USE.has(node.type))) {
const key = RootPathContextService.keyFromNode(parentKey, astNode)
// const type = astNode.type;

const foundInCache = this.cache.get(key)
const newSnippets = foundInCache ?? (await this.getSnippetsForNode(filepath, astNode))

const formattedSnippets: AutocompleteCodeSnippet[] = newSnippets.map((item) => ({
filepath: item.filepath,
content: item.contents,
type: AutocompleteSnippetType.Code,
}))

snippets.push(...formattedSnippets)
snippets.push(...newSnippets)

if (!foundInCache) {
this.cache.set(key, newSnippets)
Expand Down
11 changes: 1 addition & 10 deletions src/services/continuedev/core/autocomplete/types.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,7 @@
import { IDE, RangeInFileWithContents } from "../index"
import { IDE } from "../index"
import { AutocompleteLanguageInfo } from "./constants/AutocompleteLanguageInfo"
import { AutocompleteCodeSnippet } from "./snippets/types"

/**
* @deprecated This type should be removed in the future or renamed.
* We have a new interface called AutocompleteSnippet which is more
* general.
*/
export type AutocompleteSnippetDeprecated = RangeInFileWithContents & {
score?: number
}

export type GetLspDefinitionsFunction = (
filepath: string,
contents: string,
Expand Down