Add DatabaseClient support

public/client.js

@@ -8,6 +8,13 @@ const attachedFiles = new Map();
 const resolveFile = (name) => attachedFiles.get(name);
 main.builtin("FileAttachment", runtime.fileAttachments(resolveFile));
 
+const databaseTokens = new Map();
+async function resolveDatabaseToken(name) {
+  const token = databaseTokens.get(name);
+  if (!token) throw new Error(`Database configuration for ${name} not found`);
+  return token;
+}
+
 const cellsById = new Map();
 const Generators = library.Generators;
 
@@ -51,6 +58,7 @@ function Mutable() {
 // loading the library twice). Also, it’s nice to avoid require!
 function recommendedLibraries() {
   return {
+    DatabaseClient: () => import("./database.js").then((db) => db.makeDatabaseClient(resolveDatabaseToken)),
     d3: () => import("npm:d3"),
     htl: () => import("npm:htl"),
     html: () => import("npm:htl").then((htl) => htl.html),
@@ -71,7 +79,7 @@ function recommendedLibraries() {
 }
 
 export function define(cell) {
-  const {id, inline, inputs = [], outputs = [], files = [], body} = cell;
+  const {id, inline, inputs = [], outputs = [], files = [], databases = [], body} = cell;
   const variables = [];
   cellsById.get(id)?.variables.forEach((v) => v.delete());
   cellsById.set(id, {cell, variables});
@@ -108,6 +116,7 @@ export function define(cell) {
   variables.push(v);
   for (const o of outputs) variables.push(main.define(o, [`cell ${id}`], (exports) => exports[o]));
   for (const f of files) attachedFiles.set(f.name, {url: String(new URL(`/_file/${f.name}`, location)), mimeType: f.mimeType}); // prettier-ignore
+  for (const d of databases) databaseTokens.set(d.name, d);
 }
 
 export function open({hash} = {}) {
@@ -164,6 +173,7 @@ export function open({hash} = {}) {
                       inline: item.inline,
                       inputs: item.inputs,
                       outputs: item.outputs,
+                      databases: item.databases,
                       files: item.files,
                       body: (0, eval)(item.body)
                     });

public/database.js

@@ -0,0 +1,197 @@
+export function makeDatabaseClient(resolveToken) {
+  return function DatabaseClient(name) {
+    if (new.target !== undefined) throw new TypeError("DatabaseClient is not a constructor");
+    return resolveToken((name += "")).then((token) => new DatabaseClientImpl(name, token));
+  };
+}
+
+class DatabaseClientImpl {
+  #token;
+
+  constructor(name, token) {
+    this.name = name;
+    this.#token = token;
+  }
+  async query(sql, params, {signal} = {}) {
+    const queryUrl = new URL("/query", this.#token.url).toString();
+    const response = await fetch(queryUrl, {
+      method: "POST",
+      headers: {
+        Authorization: `Bearer ${this.#token.token}`,
+        "Content-Type": "application/json"
+      },
+      body: JSON.stringify({sql, params}),
+      signal
+    });
+    if (!response.ok) {
+      if (response.status === 401) {
+        throw new Error("Unauthorized: invalid or expired token. Try again?");
+      }
+      const contentType = response.headers.get("content-type");
+      throw new Error(
+        contentType && contentType.startsWith("application/json")
+          ? (await response.json()).message
+          : await response.text()
+      );
+    }
+    const {data, schema: jsonSchema} = await response.json();
+
+    const schema = parseJsonSchema(jsonSchema);
+    if (schema) {
+      coerce(schema, data);
+      Object.defineProperty(data, "schema", {
+        value: schema,
+        writable: true
+      });
+    }
+
+    return data;
+  }
+
+  queryTag(strings, ...args) {
+    switch (this.type) {
+      case "oracle":
+      case "databricks":
+        return [strings.reduce((prev, curr, i) => `${prev}:${i}${curr}`), args];
+      case "mysql":
+        return [strings.reduce((prev, curr, i) => `${prev}@${i}${curr}`), args];
+      case "postgres":
+        return [strings.reduce((prev, curr, i) => `${prev}$${i}${curr}`), args];
+    }
+    return [strings.join("?"), args];
+  }
+
+  async sql() {
+    return this.query(...this.queryTag.apply(this, arguments));
+  }
+}
+
+function coerceBuffer(d) {
+  return Uint8Array.from(d.data).buffer;
+}
+
+function coerceDate(d) {
+  return new Date(d);
+}
+
+function coerceBigInt(d) {
+  return BigInt(d);
+}
+
+function coercer(schema) {
+  const mappings = schema
+    .map(({name, type}) => {
+      switch (type) {
+        case "buffer":
+          return [name, coerceBuffer];
+        case "date":
+          return [name, coerceDate];
+        case "bigint":
+          return [name, coerceBigInt];
+      }
+    })
+    .filter((d) => d);
+  return (data) => {
+    for (const [column, coerce] of mappings) {
+      for (const row of data) {
+        if (row[column] != null) {
+          row[column] = coerce(row[column]);
+        }
+      }
+    }
+    return data;
+  };
+}
+
+function coerce(schema, data) {
+  return coercer(schema)(data);
+}
+
+// The data connector returns certain types as "database types" that we want to
+// treat as (JavaScript) types.
+function jsType(type, typeFlag) {
+  if (
+    (type === "string" && typeFlag === "date") ||
+    (type === "object" && typeFlag === "buffer")
+    // (type === "string" && typeFlag === "bigint") // TODO coerce bigints
+  ) {
+    return typeFlag;
+  }
+  return type;
+}
+
+function parseType(typeOptions) {
+  let type;
+  let nullable;
+  if (Array.isArray(typeOptions)) {
+    // type: ["string"] (not nullable)
+    // type: ["null", "string"] (nullable)
+    type = typeOptions.find((t) => t !== "null") ?? "other";
+    nullable = typeOptions.some((t) => t === "null");
+  } else {
+    // type: "string" (not nullable)
+    type = typeOptions;
+    nullable = false;
+  }
+  return {type, nullable};
+}
+
+/**
+ * This function parses a JSON schema object into an array of objects, matching
+ * the "column set schema" structure defined in the DatabaseClient
+ * specification. It does not support nested types (e.g. array element types,
+ * object property types), but may do so in the future.
+ * https://observablehq.com/@observablehq/database-client-specification
+ *
+ * For example, this JSON schema object:
+ * {
+ *   type: "array",
+ *   items: {
+ *     type: "object",
+ *     properties: {
+ *       TrackId: { type: "integer" },
+ *       Name: { type: "string", varchar: true },
+ *       AlbumId: { type: "number", long: true },
+ *       GenreId: { type: "array" },
+ *     }
+ *   }
+ * }
+ *
+ * will be parsed into this column set schema:
+ *
+ * [
+ *   {name: "TrackId", type: "integer"},
+ *   {name: "Name", type: "string", databaseType: "varchar"},
+ *   {name: "AlbumId", type: "number", databaseType: "long"},
+ *   {name: "GenreId", type: "array"}
+ * ]
+ */
+function parseJsonSchema(schema) {
+  if (schema?.type !== "array" || schema.items?.type !== "object" || schema.items.properties === undefined) {
+    return [];
+  }
+  return Object.entries(schema.items.properties).map(([name, {type: typeOptions, ...rest}]) => {
+    const {type, nullable} = parseType(typeOptions);
+    let typeFlag;
+
+    // The JSON Schema representation used by the data connector includes some
+    // arbitrary additional boolean properties to indicate the database type,
+    // such as {type: ["null", "string"], date: true}. This code is a little
+    // bit dangerous because the first of ANY exactly true property will be
+    // considered the database type; for example, we must be careful to ignore
+    // {type: "object", properties: {…}} and {type: "array", items: {…}}.
+    for (const key in rest) {
+      if (rest[key] === true) {
+        typeFlag = key;
+        break;
+      }
+    }
+
+    return {
+      name,
+      type: jsType(type, typeFlag),
+      nullable,
+      ...(typeFlag && {databaseType: typeFlag})
+    };
+  });
+}

src/build.ts

@@ -6,6 +6,7 @@ import {parseArgs} from "node:util";
 import {visitFiles, visitMarkdownFiles} from "./files.js";
 import {readPages} from "./navigation.js";
 import {renderServerless} from "./render.js";
+import {makeCLIResolver} from "./resolver.js";
 
 const EXTRA_FILES = new Map([["node_modules/@observablehq/runtime/dist/runtime.js", "_observablehq/runtime.js"]]);
 
@@ -20,12 +21,18 @@ async function build(context: CommandContext) {
   // Render .md files, building a list of file attachments as we go.
   const pages = await readPages(sourceRoot);
   const files: string[] = [];
+  const resolver = await makeCLIResolver();
   for await (const sourceFile of visitMarkdownFiles(sourceRoot)) {
     const sourcePath = join(sourceRoot, sourceFile);
     const outputPath = join(outputRoot, join(dirname(sourceFile), basename(sourceFile, ".md") + ".html"));
     console.log("render", sourcePath, "→", outputPath);
     const path = `/${join(dirname(sourceFile), basename(sourceFile, ".md"))}`;
-    const render = renderServerless(await readFile(sourcePath, "utf-8"), {root: sourceRoot, path, pages});
+    const render = renderServerless(await readFile(sourcePath, "utf-8"), {
+      root: sourceRoot,
+      path,
+      pages,
+      resolver
+    });
     files.push(...render.files.map((f) => join(sourceFile, "..", f.name)));
     await prepareOutput(outputPath);
     await writeFile(outputPath, render.html);

src/error.ts

@@ -1,7 +1,7 @@
 export class HttpError extends Error {
   public readonly statusCode: number;
 
-  constructor(message: string, statusCode: number, cause?: Error) {
+  constructor(message: string, statusCode: number, cause?: any) {
     super(message ?? `HTTP status ${statusCode}`, cause);
     this.statusCode = statusCode;
     Error.captureStackTrace(this, HttpError);

src/javascript.ts

@@ -11,6 +11,10 @@ import {findImports, rewriteImports} from "./javascript/imports.js";
 import {findReferences} from "./javascript/references.js";
 import {Sourcemap} from "./sourcemap.js";
 
+export interface DatabaseReference {
+  name: string;
+}
+
 export interface FileReference {
   name: string;
   mimeType: string | null;
@@ -26,6 +30,7 @@ export interface Transpile {
   outputs?: string[];
   inline?: boolean;
   body: string;
+  databases?: DatabaseReference[];
   files?: FileReference[];
   imports?: ImportReference[];
 }
@@ -42,6 +47,7 @@ export function transpileJavaScript(input: string, options: ParseOptions): Trans
   const {root, id} = options;
   try {
     const node = parseJavaScript(input, options);
+    const databases = node.features.filter((f) => f.type === "DatabaseClient").map((f) => ({name: f.name}));
     const files = node.features
       .filter((f) => f.type === "FileAttachment")
       .filter((f) => canReadSync(join(root, f.name)))
@@ -61,6 +67,7 @@ export function transpileJavaScript(input: string, options: ParseOptions): Trans
       ...(inputs.length ? {inputs} : null),
       ...(options.inline ? {inline: true} : null),
       ...(node.declarations?.length ? {outputs: node.declarations.map(({name}) => name)} : null),
+      ...(databases.length ? {databases} : null),
       ...(files.length ? {files} : null),
       body: `${node.async ? "async " : ""}(${inputs}) => {
 ${String(output)}${node.declarations?.length ? `\nreturn {${node.declarations.map(({name}) => name)}};` : ""}