updated rerank and embed logic

Author: RomneyDa

core/config/load.ts

@@ -8,7 +8,6 @@ import {
   ConfigValidationError,
   ModelRole,
 } from "@continuedev/config-yaml";
-import { fetchwithRequestOptions } from "@continuedev/fetch";
 import * as JSONC from "comment-json";
 import * as tar from "tar";
 
@@ -37,17 +36,15 @@ import {
   slashCommandFromDescription,
   slashFromCustomCommand,
 } from "../commands/index";
-import { AllRerankers } from "../context/allRerankers";
 import { MCPManagerSingleton } from "../context/mcp";
 import CodebaseContextProvider from "../context/providers/CodebaseContextProvider";
 import ContinueProxyContextProvider from "../context/providers/ContinueProxyContextProvider";
 import CustomContextProviderClass from "../context/providers/CustomContextProvider";
 import FileContextProvider from "../context/providers/FileContextProvider";
 import { contextProviderClassFromName } from "../context/providers/index";
 import { useHub } from "../control-plane/env";
-import { allEmbeddingsProviders } from "../indexing/allEmbeddingsProviders";
 import { BaseLLM } from "../llm";
-import { llmFromDescription } from "../llm/llms";
+import { LLMClasses, llmFromDescription } from "../llm/llms";
 import CustomLLMClass from "../llm/llms/CustomLLM";
 import FreeTrial from "../llm/llms/FreeTrial";
 import { LLMReranker } from "../llm/llms/llm";
@@ -420,39 +417,32 @@ async function intermediateToFinalConfig(
   function getEmbeddingsILLM(
     embedConfig: EmbeddingsProviderDescription | ILLM | undefined,
   ): ILLM | null {
-    if (!embedConfig) {
-      return null;
-    }
-    // config.ts-injected ILLM
-    if ("providerName" in embedConfig) {
-      return embedConfig;
-    }
-    const { provider, ...options } = embedConfig;
-    const embeddingsProviderClass = allEmbeddingsProviders[provider];
-    if (embeddingsProviderClass) {
-      if (
-        embeddingsProviderClass.name === "_TransformersJsEmbeddingsProvider"
-      ) {
-        return new embeddingsProviderClass();
+    if (embedConfig) {
+      // config.ts-injected ILLM
+      if ("providerName" in embedConfig) {
+        return embedConfig;
+      }
+      const { provider, ...options } = embedConfig;
+      if (provider === "transformers.js") {
+        return new TransformersJsEmbeddingsProvider();
       } else {
-        const llmOptions: LLMOptions = {
-          model: options.model ?? "UNSPECIFIED",
-          ...options,
-        };
-        return new embeddingsProviderClass(
-          llmOptions,
-          (url: string | URL, init: any) =>
-            fetchwithRequestOptions(url, init, {
-              ...config.requestOptions,
-              ...options.requestOptions,
-            }),
-        );
+        const cls = LLMClasses.find((c) => c.providerName === provider);
+        if (cls) {
+          const llmOptions: LLMOptions = {
+            model: options.model ?? "UNSPECIFIED",
+            ...options,
+          };
+          return new cls(llmOptions);
+        } else {
+          errors.push({
+            fatal: false,
+            message: `Embeddings provider ${provider} not found`,
+          });
+        }
       }
-    } else {
-      errors.push({
-        fatal: false,
-        message: `Embeddings provider ${provider} not found. Using default`,
-      });
+    }
+    if (ideInfo.ideType === "vscode") {
+      return new TransformersJsEmbeddingsProvider();
     }
     return null;
   }
@@ -470,7 +460,6 @@ async function intermediateToFinalConfig(
       return rerankingConfig;
     }
     const { name, params } = config.reranker as RerankerDescription;
-    const rerankerClass = AllRerankers[name];
 
     if (name === "llm") {
       const llm = models.find((model) => model.title === params?.modelTitle);
@@ -483,17 +472,20 @@ async function intermediateToFinalConfig(
       } else {
         return new LLMReranker(llm);
       }
-    } else if (rerankerClass) {
-      const llmOptions: LLMOptions = {
-        model: "rerank-2",
-        ...params,
-      };
-      return new rerankerClass(llmOptions, (url: string | URL, init: any) =>
-        fetchwithRequestOptions(url, init, {
-          ...config.requestOptions,
-          ...params?.requestOptions,
-        }),
-      );
+    } else {
+      const cls = LLMClasses.find((c) => c.providerName === name);
+      if (cls) {
+        const llmOptions: LLMOptions = {
+          model: params?.model,
+          ...params,
+        };
+        return new cls(llmOptions);
+      } else {
+        errors.push({
+          fatal: false,
+          message: `Unknown reranking provider ${name}`,
+        });
+      }
     }
     return null;
   }

core/config/yaml/loadYaml.ts

@@ -243,7 +243,7 @@ async function configYamlToContinueConfig(
       }
 
       if (model.roles?.includes("embed")) {
-        const { provider, ...options } = model;
+        const { provider } = model;
         if (provider === "transformers.js") {
           if (ideInfo.ideType === "vscode") {
             continueConfig.modelsByRole.embed.push(