Make search functionality more customizable

Resolves #1553
Resolves #1953

Author: Gerrit0

CHANGELOG.md

@@ -3,10 +3,14 @@
 ### Bug Fixes
 
 -   TypeDoc no longer ignores project references if `--entryPointStrategy Packages` is set, #1976.
+-   Boost computations are now done when creating the search index, resulting in a smaller `search.js` generated file.
 
 ### Features
 
 -   The `--exclude` option will now be respected by `--entryPointStrategy Packages` and can be used to exclude package directories, #1959.
+-   TypeDoc now emits an `IndexEvent` on the `Renderer` when preparing the search index, #1953.
+-   Added new `--searchInComments` option to include comment text in the search index, #1553.
+    Turning this option on will increase the size of your search index, potentially by an order of magnitude.
 
 ## v0.23.3 (2022-07-01)
 

src/index.ts

@@ -15,6 +15,7 @@ export {
     PageEvent,
     RendererEvent,
     MarkdownEvent,
+    IndexEvent,
 } from "./lib/output";
 export type { RenderTemplate, RendererHooks } from "./lib/output";
 

src/lib/output/events.ts

@@ -3,6 +3,7 @@ import * as Path from "path";
 import { Event } from "../utils/events";
 import type { ProjectReflection } from "../models/reflections/project";
 import type { RenderTemplate, UrlMapping } from "./models/UrlMapping";
+import type { DeclarationReflection } from "../models";
 
 /**
  * An event emitted by the {@link Renderer} class at the very beginning and
@@ -152,3 +153,64 @@ export class MarkdownEvent extends Event {
         this.parsedText = parsedText;
     }
 }
+
+/**
+ * An event emitted when the search index is being prepared.
+ */
+export class IndexEvent extends Event {
+    /**
+     * Triggered on the renderer when the search index is being prepared.
+     * @event
+     */
+    static readonly PREPARE_INDEX = "prepareIndex";
+
+    /**
+     * May be filtered by plugins to reduce the results available.
+     * Additional items *should not* be added to this array.
+     *
+     * If you remove an index from this array, you must also remove the
+     * same index from {@link searchFields}. The {@link removeResult} helper
+     * will do this for you.
+     */
+    searchResults: DeclarationReflection[];
+
+    /**
+     * Additional search fields to be used when creating the search index.
+     * `name` and `comment` may be specified to overwrite TypeDoc's search fields.
+     *
+     * Do not use `id` as a custom search field.
+     */
+    searchFields: Record<string, string>[];
+
+    /**
+     * Weights for the fields defined in `searchFields`. The default will weight
+     * `name` as 10x more important than comment content.
+     *
+     * If a field added to {@link searchFields} is not added to this object, it
+     * will **not** be searchable.
+     *
+     * Do not replace this object, instead, set new properties on it for custom search
+     * fields added by your plugin.
+     */
+    readonly searchFieldWeights: Record<string, number> = {
+        name: 10,
+        comment: 1,
+    };
+
+    /**
+     * Remove a search result by index.
+     */
+    removeResult(index: number) {
+        this.searchResults.splice(index, 1);
+        this.searchFields.splice(index, 1);
+    }
+
+    constructor(name: string, searchResults: DeclarationReflection[]) {
+        super(name);
+        this.searchResults = searchResults;
+        this.searchFields = Array.from(
+            { length: this.searchResults.length },
+            () => ({})
+        );
+    }
+}

src/lib/output/index.ts

@@ -1,4 +1,4 @@
-export { PageEvent, RendererEvent, MarkdownEvent } from "./events";
+export { PageEvent, RendererEvent, MarkdownEvent, IndexEvent } from "./events";
 export { UrlMapping } from "./models/UrlMapping";
 export type { RenderTemplate } from "./models/UrlMapping";
 export { Renderer } from "./renderer";

src/lib/output/plugins/JavascriptIndexPlugin.ts

@@ -2,23 +2,38 @@ import * as Path from "path";
 import { Builder, trimmer } from "lunr";
 
 import {
+    Comment,
     DeclarationReflection,
     ProjectReflection,
     ReflectionKind,
 } from "../../models";
 import { GroupPlugin } from "../../converter/plugins";
 import { Component, RendererComponent } from "../components";
-import { RendererEvent } from "../events";
-import { writeFileSync } from "../../utils";
+import { IndexEvent, RendererEvent } from "../events";
+import { BindOption, writeFileSync } from "../../utils";
 import { DefaultTheme } from "../themes/default/DefaultTheme";
 
+/**
+ * Keep this in sync with the interface in src/lib/output/themes/default/assets/typedoc/components/Search.ts
+ */
+interface SearchDocument {
+    kind: number;
+    name: string;
+    url: string;
+    classes?: string;
+    parent?: string;
+}
+
 /**
  * A plugin that exports an index of the project to a javascript file.
  *
  * The resulting javascript file can be used to build a simple search function.
  */
 @Component({ name: "javascript-index" })
 export class JavascriptIndexPlugin extends RendererComponent {
+    @BindOption("searchInComments")
+    searchComments!: boolean;
+
     /**
      * Create a new JavascriptIndexPlugin instance.
      */
@@ -39,30 +54,52 @@ export class JavascriptIndexPlugin extends RendererComponent {
             return;
         }
 
-        const rows: any[] = [];
+        const rows: SearchDocument[] = [];
         const kinds: { [K in ReflectionKind]?: string } = {};
 
-        for (const reflection of event.project.getReflectionsByKind(
-            ReflectionKind.All
+        const initialSearchResults = Object.values(
+            event.project.reflections
+        ).filter((refl) => {
+            return (
+                refl instanceof DeclarationReflection &&
+                refl.url &&
+                refl.name &&
+                !refl.flags.isExternal
+            );
+        }) as DeclarationReflection[];
+
+        const indexEvent = new IndexEvent(
+            IndexEvent.PREPARE_INDEX,
+            initialSearchResults
+        );
+
+        this.owner.trigger(indexEvent);
+
+        if (indexEvent.isDefaultPrevented) {
+            return;
+        }
+
+        const builder = new Builder();
+        builder.pipeline.add(trimmer);
+
+        builder.ref("id");
+        for (const [key, boost] of Object.entries(
+            indexEvent.searchFieldWeights
         )) {
-            if (!(reflection instanceof DeclarationReflection)) {
-                continue;
-            }
+            builder.field(key, { boost });
+        }
 
-            if (
-                !reflection.url ||
-                !reflection.name ||
-                reflection.flags.isExternal
-            ) {
+        for (const reflection of indexEvent.searchResults) {
+            if (!reflection.url) {
                 continue;
             }
 
-            let parent = reflection.parent;
             const boost = reflection.relevanceBoost ?? 1;
             if (boost <= 0) {
                 continue;
             }
 
+            let parent = reflection.parent;
             if (parent instanceof ProjectReflection) {
                 parent = undefined;
             }
@@ -73,34 +110,29 @@ export class JavascriptIndexPlugin extends RendererComponent {
                 );
             }
 
-            const row: any = {
-                id: rows.length,
+            const row: SearchDocument = {
                 kind: reflection.kind,
                 name: reflection.name,
                 url: reflection.url,
                 classes: reflection.cssClasses,
             };
 
-            if (boost !== 1) {
-                row.boost = boost;
-            }
-
             if (parent) {
                 row.parent = parent.getFullName();
             }
 
+            builder.add(
+                {
+                    name: reflection.name,
+                    comment: this.getCommentSearchText(reflection),
+                    ...indexEvent.searchFields[rows.length],
+                    id: rows.length,
+                },
+                { boost }
+            );
             rows.push(row);
         }
 
-        const builder = new Builder();
-        builder.pipeline.add(trimmer);
-
-        builder.ref("id");
-        builder.field("name", { boost: 10 });
-        builder.field("parent");
-
-        rows.forEach((row) => builder.add(row));
-
         const index = builder.build();
 
         const jsonFileName = Path.join(
@@ -120,4 +152,29 @@ export class JavascriptIndexPlugin extends RendererComponent {
             `window.searchData = JSON.parse(${JSON.stringify(jsonData)});`
         );
     }
+
+    private getCommentSearchText(reflection: DeclarationReflection) {
+        if (!this.searchComments) return;
+
+        const comments: Comment[] = [];
+        if (reflection.comment) comments.push(reflection.comment);
+        reflection.signatures?.forEach(
+            (s) => s.comment && comments.push(s.comment)
+        );
+        reflection.getSignature?.comment &&
+            comments.push(reflection.getSignature.comment);
+        reflection.setSignature?.comment &&
+            comments.push(reflection.setSignature.comment);
+
+        if (!comments.length) {
+            return;
+        }
+
+        return comments
+            .flatMap((c) => {
+                return [...c.summary, ...c.blockTags.flatMap((t) => t.content)];
+            })
+            .map((part) => part.text)
+            .join("\n");
+    }
 }

src/lib/output/renderer.ts

@@ -11,7 +11,7 @@ import * as path from "path";
 
 import type { Application } from "../application";
 import type { Theme } from "./theme";
-import { RendererEvent, PageEvent } from "./events";
+import { RendererEvent, PageEvent, IndexEvent } from "./events";
 import type { ProjectReflection } from "../models/reflections/project";
 import type { UrlMapping } from "./models/UrlMapping";
 import { writeFileSync } from "../utils/fs";
@@ -99,6 +99,10 @@ export interface RendererHooks {
  *  * {@link Renderer.EVENT_END}<br>
  *    Triggered after the renderer has written all documents. The listener receives
  *    an instance of {@link RendererEvent}.
+ *
+ * * {@link Renderer.EVENT_PREPARE_INDEX}<br>
+ *    Triggered when the JavascriptIndexPlugin is preparing the search index. Listeners receive
+ *    an instance of {@link IndexEvent}.
  */
 @Component({ name: "renderer", internal: true, childClass: RendererComponent })
 export class Renderer extends ChildableComponent<
@@ -123,6 +127,9 @@ export class Renderer extends ChildableComponent<
     /** @event */
     static readonly EVENT_END = RendererEvent.END;
 
+    /** @event */
+    static readonly EVENT_PREPARE_INDEX = IndexEvent.PREPARE_INDEX;
+
     /**
      * The theme that is used to render the documentation.
      */

src/lib/output/themes/default/assets/typedoc/components/Search.ts

@@ -1,19 +1,23 @@
 import { debounce } from "../utils/debounce";
 import { Index } from "lunr";
 
-interface IDocument {
+/**
+ * Keep this in sync with the interface in src/lib/output/plugins/JavascriptIndexPlugin.ts
+ * It's not imported because these are separate TS projects today.
+ */
+interface SearchDocument {
     id: number;
+
     kind: number;
     name: string;
     url: string;
     classes?: string;
     parent?: string;
-    boost?: number;
 }
 
 interface IData {
     kinds: { [kind: number]: string };
-    rows: IDocument[];
+    rows: SearchDocument[];
     index: object;
 }
 
@@ -168,9 +172,6 @@ function updateResults(
                 1 + 1 / (Math.abs(row.name.length - searchText.length) * 10);
         }
 
-        // boost by relevanceBoost
-        boost *= row.boost ?? 1;
-
         item.score *= boost;
     }
 

src/lib/utils/options/declaration.ts

@@ -126,6 +126,7 @@ export interface TypeDocOptionMap {
     githubPages: boolean;
     htmlLang: string;
     hideGenerator: boolean;
+    searchInComments: boolean;
     cleanOutputDir: boolean;
 
     commentStyle: typeof CommentStyle;

src/lib/utils/options/sources/typedoc.ts

@@ -270,6 +270,11 @@ export function addTypeDocOptions(options: Pick<Options, "addDeclaration">) {
         help: "Do not print the TypeDoc link at the end of the page.",
         type: ParameterType.Boolean,
     });
+    options.addDeclaration({
+        name: "searchInComments",
+        help: "If set, the search index will also include comments. This will greatly increase the size of the search index.",
+        type: ParameterType.Boolean,
+    });
     options.addDeclaration({
         name: "cleanOutputDir",
         help: "If set, TypeDoc will remove the output directory before writing output.",