[Docs Site] Build markdown.zip and upload to R2 (#22111)

Author: KianNH

.github/workflows/publish-production.yml

@@ -32,6 +32,13 @@ jobs:
         name: Deploy to Cloudflare Workers
         env:
           CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+      - name: Build vendored Markdown archive
+        env:
+          CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+        run: |
+          npx tsx bin/generate-index-md.ts
+          cd distmd && zip -r markdown.zip .
+          npx wrangler r2 object put vendored-markdown/markdown.zip --file=markdown.zip --remote
       - uses: actions/cache/save@v4
         if: always()
         with:

.gitignore

@@ -1,5 +1,6 @@
 # build output
 dist/
+distmd/
 # generated types
 .astro/
 

bin/generate-index-md.ts

@@ -0,0 +1,32 @@
+import { readFileSync, writeFileSync, mkdirSync } from "node:fs";
+
+import glob from "fast-glob";
+import { parse } from "node-html-parser";
+import { htmlToMarkdown } from "~/util/markdown";
+
+const files = await glob("dist/**/*.html");
+
+for (const file of files) {
+	const html = readFileSync(file, "utf-8");
+	const dom = parse(html);
+
+	const url = dom
+		.querySelector("link[rel='alternate'][type='text/markdown']")
+		?.getAttribute("href");
+
+	if (!url) {
+		continue;
+	}
+
+	const markdown = await htmlToMarkdown(html, url);
+
+	if (!markdown) {
+		continue;
+	}
+
+	const path = file.replace("dist/", "distmd/").replace(".html", ".md");
+
+	mkdirSync(path.split("/").slice(0, -1).join("/"), { recursive: true });
+
+	writeFileSync(path, markdown);
+}

src/components/overrides/Head.astro

@@ -182,7 +182,7 @@ head.push({
 	attrs: {
 		rel: "alternate",
 		type: "text/markdown",
-		href: Astro.url.pathname + "index.md",
+		href: Astro.site + Astro.url.pathname.slice(1) + "index.md",
 	},
 });
 

src/pages/llms.txt.ts

@@ -31,14 +31,17 @@ export const GET: APIRoute = async () => {
 		Easily build and deploy full-stack applications everywhere,
 		thanks to integrated compute, storage, and networking.
 
+		> [!TIP]
+		> An archive of Markdown files is available at https://developers.cloudflare.com/markdown.zip
+
 		${grouped
 			.map(([product, entries]) => {
 				return dedent(`
 				## ${product}
 
 				${entries
 					?.map((e) => {
-						const line = `- [${e.data.title}](https://developers.cloudflare.com/${e.id}/)`;
+						const line = `- [${e.data.title}](https://developers.cloudflare.com/${e.id}/index.md)`;
 
 						const description = e.data.description;
 

src/util/markdown.ts

@@ -39,8 +39,8 @@ export async function htmlToMarkdown(
 		description ? `description: ${description}` : [],
 		lastUpdated ? `lastUpdated: ${lastUpdated}` : [],
 		`source_url:`,
-		`  html: ${url}`,
-		`  md: ${url.replace("index.md", "")}`,
+		`  html: ${url.replace("index.md", "")}`,
+		`  md: ${url}`,
 		"---\n",
 		markdown,
 	]

worker/index.ts

@@ -12,6 +12,16 @@ const redirectsEvaluator = generateRedirectsEvaluator(redirectsFileContents, {
 
 export default class extends WorkerEntrypoint<Env> {
 	override async fetch(request: Request) {
+		if (request.url.endsWith("/markdown.zip")) {
+			const res = await this.env.VENDORED_MARKDOWN.get("markdown.zip");
+
+			return new Response(res?.body, {
+				headers: {
+					"Content-Type": "application/zip",
+				},
+			});
+		}
+
 		if (request.url.endsWith("/index.md")) {
 			const htmlUrl = request.url.replace("index.md", "");
 			const res = await this.env.ASSETS.fetch(htmlUrl, request);

worker/index.worker.test.ts

@@ -273,8 +273,8 @@ describe("Cloudflare Docs", () => {
 				description: The HTML generated by this file is used as a test fixture for our Markdown generation.
 				lastUpdated: 2025-01-01T00:00:00.000Z
 				source_url:
-				  html: http://fakehost/style-guide/fixtures/markdown/index.md
-				  md: http://fakehost/style-guide/fixtures/markdown/
+				  html: http://fakehost/style-guide/fixtures/markdown/
+				  md: http://fakehost/style-guide/fixtures/markdown/index.md
 				---
 
 				The HTML generated by this file is used as a test fixture for our Markdown generation.
@@ -327,7 +327,9 @@ describe("Cloudflare Docs", () => {
 					"link[rel='alternate'][type='text/markdown']",
 				)?.attributes.href;
 
-				expect(markdown).toBe("/workers/index.md");
+				expect(markdown).toBe(
+					"https://developers.cloudflare.com/workers/index.md",
+				);
 			});
 
 			it("og:image tag", () => {

worker/worker-configuration.d.ts

@@ -2,6 +2,7 @@
 
 interface Env {
 	ASSETS: Fetcher;
+	VENDORED_MARKDOWN: R2Bucket;
 }
 declare module "*/__redirects" {
 	const value: string;

wrangler.toml

@@ -16,4 +16,8 @@ rules = [
 directory = "./dist"
 binding = "ASSETS"
 not_found_handling = "404-page"
-run_worker_first = true
+run_worker_first = true
+
+[[r2_buckets]]
+binding = "VENDORED_MARKDOWN"
+bucket_name = "vendored-markdown"