lazy config (#695)

* lazy config

* update docs

* fix tests

* lazy readPages

---------

Co-authored-by: Philippe Rivière <[email protected]>

Author: mbostock

docs/getting-started.md

@@ -267,7 +267,7 @@ display(1 + 2);
 ```
 ````
 
-To see the new page in the sidebar, you must restart the preview server. In the terminal, use Control-C (⌃C) to kill the preview server. Then use up arrow (↑) to re-run the command to start the preview server (`npm run dev` or `yarn dev`). Lastly, reload your browser. A bit of rigamarole, but you won’t have to do it often… 😓 Upvote <a href="https://github.com/observablehq/framework/issues/645">#645</a> and <a href="https://github.com/observablehq/framework/issues/646">#646</a> if you’d like this to be better.
+To see the new page in the sidebar, reload the page.
 
 If you click on the **Weather report** link in the sidebar, it’ll take you to <http://127.0.0.1:3000/weather>, where you should see:
 

src/bin/observable.ts

@@ -179,7 +179,8 @@ try {
       const {config, root, host, port, open} = values;
       await import("../preview.js").then(async (preview) =>
         preview.preview({
-          config: await readConfig(config, root),
+          config,
+          root,
           hostname: host!,
           port: port === undefined ? undefined : +port,
           open

src/build.ts

@@ -52,7 +52,7 @@ export async function build(
 
   // Make sure all files are readable before starting to write output files.
   let pageCount = 0;
-  for await (const sourceFile of visitMarkdownFiles(root)) {
+  for (const sourceFile of visitMarkdownFiles(root)) {
     await access(join(root, sourceFile), constants.R_OK);
     pageCount++;
   }
@@ -65,7 +65,7 @@ export async function build(
   const localImports = new Set<string>();
   const globalImports = new Set<string>();
   const stylesheets = new Set<string>();
-  for await (const sourceFile of visitMarkdownFiles(root)) {
+  for (const sourceFile of visitMarkdownFiles(root)) {
     const sourcePath = join(root, sourceFile);
     const path = join("/", dirname(sourceFile), basename(sourceFile, ".md"));
     const options = {path, ...config};

src/config.ts

@@ -1,5 +1,5 @@
-import {existsSync} from "node:fs";
-import {readFile} from "node:fs/promises";
+import {existsSync, readFileSync} from "node:fs";
+import {stat} from "node:fs/promises";
 import op from "node:path";
 import {basename, dirname, join} from "node:path/posix";
 import {cwd} from "node:process";
@@ -67,25 +67,32 @@ function resolveConfig(configPath: string, root = "."): string {
   return op.join(cwd(), root, configPath);
 }
 
+// By using the modification time of the config, we ensure that we pick up any
+// changes to the config on reload.
+async function importConfig(path: string): Promise<any> {
+  const {mtimeMs} = await stat(path);
+  return (await import(`${pathToFileURL(path).href}?${mtimeMs}`)).default;
+}
+
 export async function readConfig(configPath?: string, root?: string): Promise<Config> {
   if (configPath === undefined) return readDefaultConfig(root);
-  return normalizeConfig((await import(pathToFileURL(resolveConfig(configPath, root)).href)).default, root);
+  return normalizeConfig(await importConfig(resolveConfig(configPath, root)), root);
 }
 
 export async function readDefaultConfig(root?: string): Promise<Config> {
   const jsPath = resolveConfig("observablehq.config.js", root);
-  if (existsSync(jsPath)) return normalizeConfig((await import(pathToFileURL(jsPath).href)).default, root);
+  if (existsSync(jsPath)) return normalizeConfig(await importConfig(jsPath), root);
   const tsPath = resolveConfig("observablehq.config.ts", root);
   if (!existsSync(tsPath)) return normalizeConfig(undefined, root);
   await import("tsx/esm"); // lazy tsx
-  return normalizeConfig((await import(pathToFileURL(tsPath).href)).default, root);
+  return normalizeConfig(await importConfig(tsPath), root);
 }
 
-async function readPages(root: string, md: MarkdownIt): Promise<Page[]> {
+function readPages(root: string, md: MarkdownIt): Page[] {
   const pages: Page[] = [];
-  for await (const file of visitMarkdownFiles(root)) {
+  for (const file of visitMarkdownFiles(root)) {
     if (file === "index.md" || file === "404.md") continue;
-    const source = await readFile(join(root, file), "utf8");
+    const source = readFileSync(join(root, file), "utf8");
     const parsed = parseMarkdown(source, {path: file, md});
     if (parsed?.data?.draft) continue;
     const name = basename(file, ".md");
@@ -102,7 +109,7 @@ export function setCurrentDate(date = new Date()): void {
   currentDate = date;
 }
 
-export async function normalizeConfig(spec: any = {}, defaultRoot = "docs"): Promise<Config> {
+export function normalizeConfig(spec: any = {}, defaultRoot = "docs"): Config {
   let {
     root = defaultRoot,
     output = "dist",
@@ -127,10 +134,10 @@ export async function normalizeConfig(spec: any = {}, defaultRoot = "docs"): Pro
   else if (style !== undefined) style = {path: String(style)};
   else style = {theme: (theme = normalizeTheme(theme))};
   const md = createMarkdownIt(spec);
-  let {title, pages = await readPages(root, md), pager = true, toc = true} = spec;
+  let {title, pages, pager = true, toc = true} = spec;
   if (title !== undefined) title = String(title);
-  pages = Array.from(pages, normalizePageOrSection);
-  sidebar = sidebar === undefined ? pages.length > 0 : Boolean(sidebar);
+  if (pages !== undefined) pages = Array.from(pages, normalizePageOrSection);
+  if (sidebar !== undefined) sidebar = Boolean(sidebar);
   pager = Boolean(pager);
   scripts = Array.from(scripts, normalizeScript);
   head = String(head);
@@ -140,7 +147,7 @@ export async function normalizeConfig(spec: any = {}, defaultRoot = "docs"): Pro
   deploy = deploy ? {workspace: String(deploy.workspace).replace(/^@+/, ""), project: String(deploy.project)} : null;
   search = Boolean(search);
   interpreters = normalizeInterpreters(interpreters);
-  return {
+  const config = {
     root,
     output,
     base,
@@ -159,6 +166,9 @@ export async function normalizeConfig(spec: any = {}, defaultRoot = "docs"): Pro
     md,
     loaders: new LoaderResolver({root, interpreters})
   };
+  if (pages === undefined) Object.defineProperty(config, "pages", {get: () => readPages(root, md)});
+  if (sidebar === undefined) Object.defineProperty(config, "sidebar", {get: () => config.pages.length > 0});
+  return config;
 }
 
 function normalizeBase(base: any): string {

src/deploy.ts

@@ -58,7 +58,7 @@ export interface DeployEffects extends ConfigEffects, TtyEffects, AuthEffects {
   logger: Logger;
   input: NodeJS.ReadableStream;
   output: NodeJS.WritableStream;
-  visitFiles: (root: string) => AsyncGenerator<string>;
+  visitFiles: (root: string) => Generator<string>;
   stat: (path: string) => Promise<Stats>;
 }
 

src/files.ts

@@ -1,6 +1,6 @@
 import type {Stats} from "node:fs";
-import {existsSync} from "node:fs";
-import {mkdir, readdir, stat} from "node:fs/promises";
+import {existsSync, readdirSync, statSync} from "node:fs";
+import {mkdir, stat} from "node:fs/promises";
 import op from "node:path";
 import {extname, join, normalize, relative, sep} from "node:path/posix";
 import {cwd} from "node:process";
@@ -41,23 +41,23 @@ export function getStylePath(entry: string): string {
 }
 
 /** Yields every Markdown (.md) file within the given root, recursively. */
-export async function* visitMarkdownFiles(root: string): AsyncGenerator<string> {
-  for await (const file of visitFiles(root)) {
+export function* visitMarkdownFiles(root: string): Generator<string> {
+  for (const file of visitFiles(root)) {
     if (extname(file) !== ".md") continue;
     yield file;
   }
 }
 
 /** Yields every file within the given root, recursively. */
-export async function* visitFiles(root: string): AsyncGenerator<string> {
+export function* visitFiles(root: string): Generator<string> {
   const visited = new Set<number>();
   const queue: string[] = [(root = normalize(root))];
   for (const path of queue) {
-    const status = await stat(path);
+    const status = statSync(path);
     if (status.isDirectory()) {
       if (visited.has(status.ino)) continue; // circular symlink
       visited.add(status.ino);
-      for (const entry of await readdir(path)) {
+      for (const entry of readdirSync(path)) {
         queue.push(join(path, entry));
       }
     } else {

src/preview.ts

@@ -14,6 +14,7 @@ import send from "send";
 import type {WebSocket} from "ws";
 import {WebSocketServer} from "ws";
 import type {Config} from "./config.js";
+import {readConfig} from "./config.js";
 import {HttpError, isEnoent, isHttpError, isSystemError} from "./error.js";
 import {getClientPath} from "./files.js";
 import type {FileWatchers} from "./fileWatchers.js";
@@ -32,7 +33,8 @@ import {Telemetry} from "./telemetry.js";
 import {bold, faint, green, link} from "./tty.js";
 
 export interface PreviewOptions {
-  config: Config;
+  config?: string;
+  root?: string;
   hostname: string;
   open?: boolean;
   port?: number;
@@ -44,13 +46,25 @@ export async function preview(options: PreviewOptions): Promise<PreviewServer> {
 }
 
 export class PreviewServer {
-  private readonly _config: Config;
+  private readonly _config: string | undefined;
+  private readonly _root: string | undefined;
   private readonly _server: ReturnType<typeof createServer>;
   private readonly _socketServer: WebSocketServer;
   private readonly _verbose: boolean;
 
-  private constructor({config, server, verbose}: {config: Config; server: Server; verbose: boolean}) {
+  private constructor({
+    config,
+    root,
+    server,
+    verbose
+  }: {
+    config?: string;
+    root?: string;
+    server: Server;
+    verbose: boolean;
+  }) {
     this._config = config;
+    this._root = root;
     this._verbose = verbose;
     this._server = server;
     this._server.on("request", this._handleRequest);
@@ -86,8 +100,12 @@ export class PreviewServer {
     return new PreviewServer({server, verbose, ...options});
   }
 
+  async _readConfig() {
+    return readConfig(this._config, this._root);
+  }
+
   _handleRequest: RequestListener = async (req, res) => {
-    const config = this._config;
+    const config = await this._readConfig();
     const {root, loaders} = config;
     if (this._verbose) console.log(faint(req.method!), req.url);
     try {
@@ -207,7 +225,7 @@ export class PreviewServer {
 
   _handleConnection = async (socket: WebSocket, req: IncomingMessage) => {
     if (req.url === "/_observablehq") {
-      handleWatch(socket, req, this._config);
+      handleWatch(socket, req, await this._readConfig());
     } else {
       socket.close();
     }

src/search.ts

@@ -40,7 +40,7 @@ export async function searchIndex(config: Config, effects = defaultEffects): Pro
 
   // Index the pages
   const index = new MiniSearch(indexOptions);
-  for await (const file of visitMarkdownFiles(root)) {
+  for (const file of visitMarkdownFiles(root)) {
     const sourcePath = join(root, file);
     const source = await readFile(sourcePath, "utf8");
     const path = `/${join(dirname(file), basename(file, ".md"))}`;

test/config-test.ts

@@ -63,27 +63,27 @@ describe("readConfig(undefined, root)", () => {
 
 describe("normalizeConfig(spec, root)", () => {
   const root = "test/input/build/config";
-  it("coerces the title to a string", async () => {
-    assert.strictEqual((await config({title: 42, pages: []}, root)).title, "42");
-    assert.strictEqual((await config({title: null, pages: []}, root)).title, "null");
+  it("coerces the title to a string", () => {
+    assert.strictEqual(config({title: 42, pages: []}, root).title, "42");
+    assert.strictEqual(config({title: null, pages: []}, root).title, "null");
   });
-  it("considers the title optional", async () => {
-    assert.strictEqual((await config({pages: []}, root)).title, undefined);
-    assert.strictEqual((await config({title: undefined, pages: []}, root)).title, undefined);
+  it("considers the title optional", () => {
+    assert.strictEqual(config({pages: []}, root).title, undefined);
+    assert.strictEqual(config({title: undefined, pages: []}, root).title, undefined);
   });
-  it("populates default pages", async () => {
-    assert.deepStrictEqual((await config({}, root)).pages, [
+  it("populates default pages", () => {
+    assert.deepStrictEqual(config({}, root).pages, [
       {name: "One", path: "/one"},
       {name: "H1: Section", path: "/toc-override"},
       {name: "H1: Section", path: "/toc"},
       {name: "A page…", path: "/closed/page"},
       {name: "Two", path: "/sub/two"}
     ]);
   });
-  it("coerces pages to an array", async () => {
-    assert.deepStrictEqual((await config({pages: new Set()}, root)).pages, []);
+  it("coerces pages to an array", () => {
+    assert.deepStrictEqual(config({pages: new Set()}, root).pages, []);
   });
-  it("coerces and normalizes page paths", async () => {
+  it("coerces and normalizes page paths", () => {
     const inpages = [
       {name: 42, path: true},
       {name: null, path: {toString: () => "yes"}},
@@ -98,13 +98,13 @@ describe("normalizeConfig(spec, root)", () => {
       {name: "Index.html", path: "/foo/index"},
       {name: "Page.html", path: "/foo"}
     ];
-    assert.deepStrictEqual((await config({pages: inpages}, root)).pages, outpages);
+    assert.deepStrictEqual(config({pages: inpages}, root).pages, outpages);
   });
-  it("allows external page paths", async () => {
+  it("allows external page paths", () => {
     const pages = [{name: "Example.com", path: "https://example.com"}];
-    assert.deepStrictEqual((await config({pages}, root)).pages, pages);
+    assert.deepStrictEqual(config({pages}, root).pages, pages);
   });
-  it("allows page paths to have query strings and anchor fragments", async () => {
+  it("allows page paths to have query strings and anchor fragments", () => {
     const inpages = [
       {name: "Anchor fragment on index", path: "/test/index#foo=bar"},
       {name: "Anchor fragment on index.html", path: "/test/index.html#foo=bar"},
@@ -129,59 +129,53 @@ describe("normalizeConfig(spec, root)", () => {
       {name: "Query string on slash", path: "/test/index?foo=bar"},
       {name: "Query string", path: "/test?foo=bar"}
     ];
-    assert.deepStrictEqual((await config({pages: inpages}, root)).pages, outpages);
+    assert.deepStrictEqual(config({pages: inpages}, root).pages, outpages);
   });
-  it("coerces sections", async () => {
+  it("coerces sections", () => {
     const inpages = [{name: 42, pages: new Set([{name: null, path: {toString: () => "yes"}}])}];
     const outpages = [{name: "42", open: true, pages: [{name: "null", path: "/yes"}]}];
-    assert.deepStrictEqual((await config({pages: inpages}, root)).pages, outpages);
+    assert.deepStrictEqual(config({pages: inpages}, root).pages, outpages);
   });
-  it("coerces toc", async () => {
-    assert.deepStrictEqual((await config({pages: [], toc: {}}, root)).toc, {label: "Contents", show: true});
-    assert.deepStrictEqual((await config({pages: [], toc: {label: null}}, root)).toc, {label: "null", show: true});
+  it("coerces toc", () => {
+    assert.deepStrictEqual(config({pages: [], toc: {}}, root).toc, {label: "Contents", show: true});
+    assert.deepStrictEqual(config({pages: [], toc: {label: null}}, root).toc, {label: "null", show: true});
   });
-  it("populates default toc", async () => {
-    assert.deepStrictEqual((await config({pages: []}, root)).toc, {label: "Contents", show: true});
+  it("populates default toc", () => {
+    assert.deepStrictEqual(config({pages: []}, root).toc, {label: "Contents", show: true});
   });
-  it("promotes boolean toc to toc.show", async () => {
-    assert.deepStrictEqual((await config({pages: [], toc: true}, root)).toc, {label: "Contents", show: true});
-    assert.deepStrictEqual((await config({pages: [], toc: false}, root)).toc, {label: "Contents", show: false});
+  it("promotes boolean toc to toc.show", () => {
+    assert.deepStrictEqual(config({pages: [], toc: true}, root).toc, {label: "Contents", show: true});
+    assert.deepStrictEqual(config({pages: [], toc: false}, root).toc, {label: "Contents", show: false});
   });
-  it("coerces pager", async () => {
-    assert.strictEqual((await config({pages: [], pager: 0}, root)).pager, false);
-    assert.strictEqual((await config({pages: [], pager: 1}, root)).pager, true);
-    assert.strictEqual((await config({pages: [], pager: ""}, root)).pager, false);
-    assert.strictEqual((await config({pages: [], pager: "0"}, root)).pager, true);
+  it("coerces pager", () => {
+    assert.strictEqual(config({pages: [], pager: 0}, root).pager, false);
+    assert.strictEqual(config({pages: [], pager: 1}, root).pager, true);
+    assert.strictEqual(config({pages: [], pager: ""}, root).pager, false);
+    assert.strictEqual(config({pages: [], pager: "0"}, root).pager, true);
   });
-  it("populates default pager", async () => {
-    assert.strictEqual((await config({pages: []}, root)).pager, true);
+  it("populates default pager", () => {
+    assert.strictEqual(config({pages: []}, root).pager, true);
   });
   describe("deploy", () => {
-    it("considers deploy optional", async () => {
-      assert.strictEqual((await config({pages: []}, root)).deploy, null);
+    it("considers deploy optional", () => {
+      assert.strictEqual(config({pages: []}, root).deploy, null);
     });
-    it("coerces workspace", async () => {
-      assert.strictEqual(
-        (await config({pages: [], deploy: {workspace: 538, project: "bi"}}, root)).deploy?.workspace,
-        "538"
-      );
+    it("coerces workspace", () => {
+      assert.strictEqual(config({pages: [], deploy: {workspace: 538, project: "bi"}}, root).deploy?.workspace, "538");
     });
-    it("strips leading @ from workspace", async () => {
-      assert.strictEqual((await config({pages: [], deploy: {workspace: "@acme"}}, root)).deploy?.workspace, "acme");
+    it("strips leading @ from workspace", () => {
+      assert.strictEqual(config({pages: [], deploy: {workspace: "@acme"}}, root).deploy?.workspace, "acme");
     });
-    it("coerces project", async () => {
-      assert.strictEqual(
-        (await config({pages: [], deploy: {workspace: "adams", project: 42}}, root)).deploy?.project,
-        "42"
-      );
+    it("coerces project", () => {
+      assert.strictEqual(config({pages: [], deploy: {workspace: "adams", project: 42}}, root).deploy?.project, "42");
     });
   });
 });
 
 describe("mergeToc(spec, toc)", () => {
   const root = "test/input/build/config";
   it("merges page- and project-level toc config", async () => {
-    const toc = (await config({pages: [], toc: true}, root)).toc;
+    const toc = config({pages: [], toc: true}, root).toc;
     assert.deepStrictEqual(mergeToc({show: false}, toc), {label: "Contents", show: false});
     assert.deepStrictEqual(mergeToc({label: "On this page"}, toc), {label: "On this page", show: true});
     assert.deepStrictEqual(mergeToc(false, toc), {label: "Contents", show: false});