only include needed bundles

src/build.ts

@@ -63,8 +63,9 @@ export async function build(
   effects.logger.log(`${faint("found")} ${pageCount} ${faint(`page${pageCount === 1 ? "" : "s"} in`)} ${root}`);
 
   // Render .md files, building a list of file attachments as we go.
-  const files: string[] = [];
-  const imports: string[] = [];
+  const files = new Set<string>();
+  const localImports = new Set<string>();
+  const globalImports = new Set<string>();
   const styles: Style[] = [];
   for await (const sourceFile of visitMarkdownFiles(root)) {
     const sourcePath = join(root, sourceFile);
@@ -73,8 +74,21 @@ export async function build(
     const path = join("/", dirname(sourceFile), basename(sourceFile, ".md"));
     const render = await renderServerless(sourcePath, {path, ...config});
     const resolveFile = ({name}) => resolvePath(sourceFile, name);
-    files.push(...render.files.map(resolveFile));
-    imports.push(...render.imports.filter((i) => i.type === "local").map(resolveFile));
+    render.files.map(resolveFile).forEach(files.add, files);
+    render.imports.filter((i) => i.type === "local").forEach((i) => localImports.add(resolveFile(i)));
+    render.imports.filter((i) => i.type === "global").forEach((i) => globalImports.add(i.name));
+    if (render.inputs.includes("FileAttachment")) {
+      // TODO If you have a FileAttachment, you might also need to preload
+      // - npm:d3-dsv if you call FileAttachment.csv or FileAttachment.tsv
+      // - npm:apache-arrow if you call FileAttachment.arrow or FileAttachment.parquet
+      // - npm:parquet-wasm if you call FileAttachment.parquet
+      // - npm:@observablehq/sqlite if you call FileAttachment.sqlite
+      // - npm:@observablehq/zip if you call FileAttachment.zip
+      // - npm:@observablehq/xlsx if you call FileAttachment.xlsx
+      globalImports.add("npm:@observablehq/sqlite");
+      globalImports.add("npm:@observablehq/zip");
+      globalImports.add("npm:@observablehq/xlsx");
+    }
     await effects.writeFile(outputPath, render.html);
     const style = mergeStyle(path, render.data?.style, render.data?.theme, config.style);
     if (style && !styles.some((s) => styleEquals(s, style))) styles.push(style);
@@ -83,29 +97,27 @@ export async function build(
   // Add imported local scripts.
   for (const script of config.scripts) {
     if (!/^\w+:/.test(script.src)) {
-      imports.push(script.src);
+      localImports.add(script.src);
     }
   }
 
   // Generate the client bundles.
   if (addPublic) {
-    for (const [entry, name] of [
-      [clientEntry, "client.js"],
-      ["./src/client/stdlib.js", "stdlib.js"],
-      // TODO Prune this list based on which libraries are actually used.
-      // TODO Remove library helpers (e.g., duckdb) when they are published to npm.
-      ["./src/client/stdlib/dot.js", "stdlib/dot.js"],
-      ["./src/client/stdlib/duckdb.js", "stdlib/duckdb.js"],
-      ["./src/client/stdlib/inputs.css", "stdlib/inputs.css"],
-      ["./src/client/stdlib/inputs.js", "stdlib/inputs.js"],
-      ["./src/client/stdlib/mermaid.js", "stdlib/mermaid.js"],
-      ["./src/client/stdlib/sqlite.js", "stdlib/sqlite.js"],
-      ["./src/client/stdlib/tex.js", "stdlib/tex.js"],
-      ["./src/client/stdlib/vega-lite.js", "stdlib/vega-lite.js"],
-      ["./src/client/stdlib/xlsx.js", "stdlib/xlsx.js"],
-      ["./src/client/stdlib/zip.js", "stdlib/zip.js"],
-      ...(config.search ? [["./src/client/search.js", "search.js"]] : [])
-    ]) {
+    const bundles: [entry: string, name: string][] = [];
+    bundles.push([clientEntry, "client.js"]);
+    bundles.push(["./src/client/stdlib.js", "stdlib.js"]);
+    if (config.search) bundles.push(["./src/client/search.js", "search.js"]);
+    for (const lib of ["dot", "duckdb", "inputs", "mermaid", "sqlite", "tex", "vega-lite", "xlsx", "zip"]) {
+      if (globalImports.has(`npm:@observablehq/${lib}`)) {
+        bundles.push([`./src/client/stdlib/${lib}.js`, `stdlib/${lib}.js`]);
+      }
+    }
+    for (const lib of ["inputs"]) {
+      if (globalImports.has(`npm:@observablehq/${lib}`)) {
+        bundles.push([`./src/client/stdlib/${lib}.css`, `stdlib/${lib}.css`]);
+      }
+    }
+    for (const [entry, name] of bundles) {
       const clientPath = getClientPath(entry);
       const outputPath = join("_observablehq", name);
       effects.output.write(`${faint("bundle")} ${clientPath} ${faint("→")} `);
@@ -159,7 +171,7 @@ export async function build(
 
   // Copy over the imported modules.
   const importResolver = createImportResolver(root);
-  for (const file of imports) {
+  for (const file of localImports) {
     const sourcePath = join(root, file);
     const outputPath = join("_import", file);
     if (!existsSync(sourcePath)) {

src/javascript/features.ts

@@ -10,6 +10,9 @@ export function findFeatures(node: Node, path: string, references: Identifier[],
   const featureMap = getFeatureReferenceMap(node);
   const features: Feature[] = [];
 
+  // TODO If the FileAttachment is part of a member expression, we should be
+  // able to tell which method they’re calling on the file attachment, and thus
+  // determine which bundles need to be included in the generated build.
   simple(node, {
     CallExpression(node) {
       const {callee} = node;

src/libraries.ts

@@ -1,10 +1,11 @@
 import {resolveNpmImport} from "./javascript/imports.js";
 
-export function getImplicitSpecifiers(inputs: Set<string>): Set<string> {
+export function getImplicitSpecifiers(inputs: Iterable<string>): Set<string> {
   return addImplicitSpecifiers(new Set(), inputs);
 }
 
-export function addImplicitSpecifiers(specifiers: Set<string>, inputs: Set<string>): typeof specifiers {
+export function addImplicitSpecifiers(specifiers: Set<string>, iterable: Iterable<string>): typeof specifiers {
+  const inputs = new Set(iterable);
   if (inputs.has("d3")) specifiers.add("npm:d3");
   if (inputs.has("Plot")) specifiers.add("npm:d3").add("npm:@observablehq/plot");
   if (inputs.has("htl") || inputs.has("html") || inputs.has("svg")) specifiers.add("npm:htl");

src/markdown.ts

@@ -16,6 +16,7 @@ import {computeHash} from "./hash.js";
 import {parseInfo} from "./info.js";
 import type {FileReference, ImportReference, PendingTranspile, Transpile} from "./javascript.js";
 import {transpileJavaScript} from "./javascript.js";
+import {getImplicitSpecifiers} from "./libraries.js";
 import {transpileTag} from "./tag.js";
 import {resolvePath} from "./url.js";
 
@@ -38,6 +39,7 @@ export interface ParseResult {
   data: {[key: string]: any} | null;
   files: FileReference[];
   imports: ImportReference[];
+  inputs: string[];
   pieces: HtmlPiece[];
   cells: CellPiece[];
   hash: string;
@@ -431,18 +433,47 @@ export async function parseMarkdown(sourcePath: string, {root, path}: ParseOptio
   const context: ParseContext = {files: [], imports: [], pieces: [], startLine: 0, currentLine: 0};
   const tokens = md.parse(parts.content, context);
   const html = md.renderer.render(tokens, md.options, context); // Note: mutates context.pieces, context.files!
+  const cells = await toParseCells(context.pieces);
+  const inputs = findUnboundInputs(cells);
+  const imports = context.imports;
+  for (const name of getImplicitSpecifiers(inputs)) imports.push({type: "global", name});
   return {
     html,
     data: isEmpty(parts.data) ? null : parts.data,
     title: parts.data?.title ?? findTitle(tokens) ?? null,
     files: context.files,
-    imports: context.imports,
+    imports,
+    inputs,
     pieces: toParsePieces(context.pieces),
-    cells: await toParseCells(context.pieces),
+    cells,
     hash: await computeMarkdownHash(source, root, path, context.imports)
   };
 }
 
+// Returns any inputs that are not declared in outputs. These typically refer to
+// symbols provided by the standard library, such as d3 and Inputs.
+function findUnboundInputs(cells: CellPiece[]): string[] {
+  const outputs = new Set<string>();
+  const inputs = new Set<string>();
+  for (const cell of cells) {
+    if (cell.outputs) {
+      for (const output of cell.outputs) {
+        outputs.add(output);
+      }
+    }
+  }
+  for (const cell of cells) {
+    if (cell.inputs) {
+      for (const input of cell.inputs) {
+        if (!outputs.has(input)) {
+          inputs.add(input);
+        }
+      }
+    }
+  }
+  return Array.from(inputs);
+}
+
 async function computeMarkdownHash(
   contents: string,
   root: string,

src/render.ts

@@ -6,7 +6,7 @@ import {type Html, html} from "./html.js";
 import type {ImportResolver} from "./javascript/imports.js";
 import {createImportResolver, resolveModuleIntegrity, resolveModulePreloads} from "./javascript/imports.js";
 import type {FileReference, ImportReference, Transpile} from "./javascript.js";
-import {addImplicitSpecifiers, addImplicitStylesheets} from "./libraries.js";
+import {addImplicitStylesheets} from "./libraries.js";
 import {type ParseResult, parseMarkdown} from "./markdown.js";
 import {type PageLink, findLink, normalizePath} from "./pager.js";
 import {getPreviewStylesheet} from "./preview.js";
@@ -17,6 +17,7 @@ export interface Render {
   html: string;
   files: FileReference[];
   imports: ImportReference[];
+  inputs: string[];
   data: ParseResult["data"];
 }
 
@@ -31,6 +32,7 @@ export async function renderPreview(sourcePath: string, options: RenderOptions):
     html: await render(parseResult, {...options, preview: true}),
     files: parseResult.files,
     imports: parseResult.imports,
+    inputs: parseResult.inputs,
     data: parseResult.data
   };
 }
@@ -41,6 +43,7 @@ export async function renderServerless(sourcePath: string, options: RenderOption
     html: await render(parseResult, options),
     files: parseResult.files,
     imports: parseResult.imports,
+    inputs: parseResult.inputs,
     data: parseResult.data
   };
 }
@@ -200,8 +203,6 @@ async function renderHead(
   if (style) stylesheets.add(style);
   const specifiers = new Set<string>(["npm:@observablehq/runtime", "npm:@observablehq/stdlib"]);
   for (const {name} of parseResult.imports) specifiers.add(name);
-  const inputs = new Set(parseResult.cells.flatMap((cell) => cell.inputs ?? []));
-  addImplicitSpecifiers(specifiers, inputs);
   await addImplicitStylesheets(stylesheets, specifiers);
   const preloads = new Set<string>([relativeUrl(path, "/_observablehq/client.js")]);
   for (const specifier of specifiers) preloads.add(await resolver(path, specifier));

test/deploy-test.ts

@@ -21,26 +21,6 @@ import {
 import {MockAuthEffects} from "./observableApiAuth-test.js";
 import {MockConfigEffects} from "./observableApiConfig-test.js";
 
-// These files are implicitly generated. This may change over time, so they’re
-// enumerated here for clarity. TODO We should enforce that these files are
-// specifically uploaded, rather than just the number of files.
-const EXTRA_FILES: string[] = [
-  "_observablehq/client.js",
-  "_observablehq/runtime.js",
-  "_observablehq/stdlib.js",
-  "_observablehq/stdlib/dot.js",
-  "_observablehq/stdlib/duckdb.js",
-  "_observablehq/stdlib/inputs.css",
-  "_observablehq/stdlib/inputs.js",
-  "_observablehq/stdlib/mermaid.js",
-  "_observablehq/stdlib/sqlite.js",
-  "_observablehq/stdlib/tex.js",
-  "_observablehq/stdlib/vega-lite.js",
-  "_observablehq/stdlib/xlsx.js",
-  "_observablehq/stdlib/zip.js",
-  "_observablehq/style.css"
-];
-
 interface MockDeployEffectsOptions {
   apiKey?: string | null;
   deployConfig?: DeployConfig | null;
@@ -148,7 +128,11 @@ describe("deploy", () => {
       .handleGetCurrentUser()
       .handleGetProject(DEPLOY_CONFIG)
       .handlePostDeploy({projectId: DEPLOY_CONFIG.projectId, deployId})
-      .handlePostDeployFile({deployId, repeat: EXTRA_FILES.length + 1})
+      .handlePostDeployFile({deployId, clientName: "index.html"})
+      .handlePostDeployFile({deployId, clientName: "_observablehq/theme-air,near-midnight.css"})
+      .handlePostDeployFile({deployId, clientName: "_observablehq/client.js"})
+      .handlePostDeployFile({deployId, clientName: "_observablehq/runtime.js"})
+      .handlePostDeployFile({deployId, clientName: "_observablehq/stdlib.js"})
       .handlePostDeployUploaded({deployId})
       .handleGetDeploy({deployId, deployStatus: "uploaded"})
       .start();
@@ -165,13 +149,14 @@ describe("deploy", () => {
     const oldTitle = `${TEST_CONFIG.title!} old`;
     getCurrentObservableApi()
       .handleGetCurrentUser()
-      .handleGetProject({
-        ...DEPLOY_CONFIG,
-        title: oldTitle
-      })
+      .handleGetProject({...DEPLOY_CONFIG, title: oldTitle})
       .handleUpdateProject({projectId: DEPLOY_CONFIG.projectId, title: TEST_CONFIG.title!})
       .handlePostDeploy({projectId: DEPLOY_CONFIG.projectId, deployId})
-      .handlePostDeployFile({deployId, repeat: EXTRA_FILES.length + 1})
+      .handlePostDeployFile({deployId, clientName: "index.html"})
+      .handlePostDeployFile({deployId, clientName: "_observablehq/theme-air,near-midnight.css"})
+      .handlePostDeployFile({deployId, clientName: "_observablehq/client.js"})
+      .handlePostDeployFile({deployId, clientName: "_observablehq/runtime.js"})
+      .handlePostDeployFile({deployId, clientName: "_observablehq/stdlib.js"})
       .handlePostDeployUploaded({deployId})
       .handleGetDeploy({deployId})
       .start();
@@ -191,7 +176,11 @@ describe("deploy", () => {
       .handleGetCurrentUser()
       .handleGetProject(deployConfig)
       .handlePostDeploy({projectId: deployConfig.projectId, deployId})
-      .handlePostDeployFile({deployId, repeat: EXTRA_FILES.length + 1})
+      .handlePostDeployFile({deployId, clientName: "index.html"})
+      .handlePostDeployFile({deployId, clientName: "_observablehq/theme-air,near-midnight.css"})
+      .handlePostDeployFile({deployId, clientName: "_observablehq/client.js"})
+      .handlePostDeployFile({deployId, clientName: "_observablehq/runtime.js"})
+      .handlePostDeployFile({deployId, clientName: "_observablehq/stdlib.js"})
       .handlePostDeployUploaded({deployId})
       .handleGetDeploy({deployId})
       .start();
@@ -214,7 +203,11 @@ describe("deploy", () => {
       })
       .handlePostProject({projectId: DEPLOY_CONFIG.projectId})
       .handlePostDeploy({projectId: DEPLOY_CONFIG.projectId, deployId})
-      .handlePostDeployFile({deployId, repeat: EXTRA_FILES.length + 1})
+      .handlePostDeployFile({deployId, clientName: "index.html"})
+      .handlePostDeployFile({deployId, clientName: "_observablehq/theme-air,near-midnight.css"})
+      .handlePostDeployFile({deployId, clientName: "_observablehq/client.js"})
+      .handlePostDeployFile({deployId, clientName: "_observablehq/runtime.js"})
+      .handlePostDeployFile({deployId, clientName: "_observablehq/stdlib.js"})
       .handlePostDeployUploaded({deployId})
       .handleGetDeploy({deployId})
       .start();
@@ -443,7 +436,11 @@ describe("deploy", () => {
       .handleGetCurrentUser()
       .handleGetProject(DEPLOY_CONFIG)
       .handlePostDeploy({projectId: DEPLOY_CONFIG.projectId, deployId})
-      .handlePostDeployFile({deployId, repeat: EXTRA_FILES.length + 1})
+      .handlePostDeployFile({deployId, clientName: "index.html"})
+      .handlePostDeployFile({deployId, clientName: "_observablehq/theme-air,near-midnight.css"})
+      .handlePostDeployFile({deployId, clientName: "_observablehq/client.js"})
+      .handlePostDeployFile({deployId, clientName: "_observablehq/runtime.js"})
+      .handlePostDeployFile({deployId, clientName: "_observablehq/stdlib.js"})
       .handlePostDeployUploaded({deployId, status: 500})
       .start();
     const effects = new MockDeployEffects({deployConfig: DEPLOY_CONFIG});
@@ -548,7 +545,11 @@ describe("deploy", () => {
           projectId: newProjectId
         })
         .handlePostDeploy({projectId: newProjectId, deployId})
-        .handlePostDeployFile({deployId, repeat: EXTRA_FILES.length + 1})
+        .handlePostDeployFile({deployId, clientName: "index.html"})
+        .handlePostDeployFile({deployId, clientName: "_observablehq/theme-air,near-midnight.css"})
+        .handlePostDeployFile({deployId, clientName: "_observablehq/client.js"})
+        .handlePostDeployFile({deployId, clientName: "_observablehq/runtime.js"})
+        .handlePostDeployFile({deployId, clientName: "_observablehq/stdlib.js"})
         .handlePostDeployUploaded({deployId})
         .handleGetDeploy({deployId})
         .start();
@@ -632,7 +633,11 @@ describe("deploy", () => {
       .handleGetCurrentUser()
       .handleGetProject(DEPLOY_CONFIG)
       .handlePostDeploy({projectId: DEPLOY_CONFIG.projectId, deployId})
-      .handlePostDeployFile({deployId, repeat: EXTRA_FILES.length + 1})
+      .handlePostDeployFile({deployId, clientName: "index.html"})
+      .handlePostDeployFile({deployId, clientName: "_observablehq/theme-air,near-midnight.css"})
+      .handlePostDeployFile({deployId, clientName: "_observablehq/client.js"})
+      .handlePostDeployFile({deployId, clientName: "_observablehq/runtime.js"})
+      .handlePostDeployFile({deployId, clientName: "_observablehq/stdlib.js"})
       .handlePostDeployUploaded({deployId})
       .handleGetDeploy({deployId})
       .start();

test/mocks/observableApi.ts

@@ -215,14 +215,20 @@ class ObservableApiMock {
 
   handlePostDeployFile({
     deployId,
+    clientName,
     status = 204,
     repeat = 1
-  }: {deployId?: string; status?: number; repeat?: number} = {}): ObservableApiMock {
+  }: {deployId?: string; clientName?: string; status?: number; repeat?: number} = {}): ObservableApiMock {
     const response = status == 204 ? "" : emptyErrorBody;
     const headers = authorizationHeader(status !== 403);
     this.addHandler((pool) => {
       pool
-        .intercept({path: `/cli/deploy/${deployId}/file`, method: "POST", headers: headersMatcher(headers)})
+        .intercept({
+          path: `/cli/deploy/${deployId}/file`,
+          method: "POST",
+          headers: headersMatcher(headers),
+          body: clientName === undefined ? undefined : formDataMatcher({client_name: clientName})
+        })
         .reply(status, response)
         .times(repeat);
     });
@@ -327,6 +333,18 @@ function headersMatcher(expected: Record<string, string | RegExp>): (headers: Re
   };
 }
 
+function formDataMatcher(expected: Record<string, string>): (body: string) => boolean {
+  // actually FormData, not string
+  return (actual: any) => {
+    for (const key in expected) {
+      if (!(actual.get(key) === expected[key])) {
+        return false;
+      }
+    }
+    return true;
+  };
+}
+
 const userBase = {
   id: "0000000000000000",
   login: "mock-user",

test/output/block-expression.json

@@ -3,6 +3,9 @@
   "title": null,
   "files": [],
   "imports": [],
+  "inputs": [
+    "display"
+  ],
   "pieces": [
     {
       "type": "html",