feat: support compile and extract with multi thread

packages/cli/package.json

@@ -81,14 +81,16 @@
     "picocolors": "^1.1.1",
     "pofile": "^1.1.4",
     "pseudolocale": "^2.0.0",
-    "source-map": "^0.8.0-beta.0"
+    "source-map": "^0.8.0-beta.0",
+    "threads": "^1.7.0"
   },
   "devDependencies": {
     "@lingui/jest-mocks": "*",
     "@types/convert-source-map": "^2.0.0",
     "@types/micromatch": "^4.0.1",
     "@types/normalize-path": "^3.0.0",
     "mock-fs": "^5.2.0",
-    "mockdate": "^3.0.5"
+    "mockdate": "^3.0.5",
+    "ts-node": "^10.9.2"
   }
 }

packages/cli/src/api/ProgramExit.ts

@@ -0,0 +1,7 @@
+export class ProgramExit extends Error {
+  constructor() {
+    super()
+
+    this.name = "ProgramExit"
+  }
+}

packages/cli/src/api/catalog.test.ts

@@ -4,7 +4,7 @@ import mockFs from "mock-fs"
 import { mockConsole } from "@lingui/jest-mocks"
 import { LinguiConfig, makeConfig } from "@lingui/conf"
 
-import { Catalog, cleanObsolete, order } from "./catalog"
+import { Catalog, cleanObsolete, order, writeCompiled } from "./catalog"
 import { createCompiledCatalog } from "./compile"
 
 import {
@@ -688,9 +688,9 @@ describe("writeCompiled", () => {
     async ({ namespace, extension }) => {
       const { source } = createCompiledCatalog("en", {}, { namespace })
       // Test that the file extension of the compiled catalog is `.mjs`
-      expect(await catalog.writeCompiled("en", source, namespace)).toMatch(
-        extension
-      )
+      expect(
+        await writeCompiled(catalog.path, "en", source, namespace)
+      ).toMatch(extension)
     }
   )
 })

packages/cli/src/api/catalog.ts

@@ -12,7 +12,6 @@ import { CompiledCatalogNamespace } from "./compile"
 import {
   getTranslationsForCatalog,
   GetTranslationsOptions,
-  TranslationMissingEvent,
 } from "./catalog/getTranslationsForCatalog"
 import { mergeCatalog } from "./catalog/mergeCatalog"
 import { extractFromFiles } from "./catalog/extractFromFiles"
@@ -181,23 +180,8 @@ export class Catalog {
     )
   }
 
-  async getTranslations(
-    locale: string,
-    options: Omit<GetTranslationsOptions, "onMissing">
-  ) {
-    const missing: TranslationMissingEvent[] = []
-
-    const messages = await getTranslationsForCatalog(this, locale, {
-      ...options,
-      onMissing: (event) => {
-        missing.push(event)
-      },
-    })
-
-    return {
-      missing,
-      messages,
-    }
+  async getTranslations(locale: string, options: GetTranslationsOptions) {
+    return await getTranslationsForCatalog(this, locale, options)
   }
 
   async write(
@@ -217,29 +201,6 @@ export class Catalog {
     await this.format.write(filename, messages, undefined)
   }
 
-  async writeCompiled(
-    locale: string,
-    compiledCatalog: string,
-    namespace?: CompiledCatalogNamespace
-  ) {
-    let ext: string
-    switch (namespace) {
-      case "es":
-        ext = "mjs"
-        break
-      case "ts":
-      case "json":
-        ext = namespace
-        break
-      default:
-        ext = "js"
-    }
-
-    const filename = `${replacePlaceholders(this.path, { locale })}.${ext}`
-    await writeFile(filename, compiledCatalog)
-    return filename
-  }
-
   async read(locale: string): Promise<CatalogType> {
     return await this.format.read(this.getFilename(locale), locale)
   }
@@ -364,6 +325,30 @@ function orderByOrigin<T extends ExtractedCatalogType>(messages: T): T {
     }, {} as T)
 }
 
+export async function writeCompiled(
+  path: string,
+  locale: string,
+  compiledCatalog: string,
+  namespace?: CompiledCatalogNamespace
+) {
+  let ext: string
+  switch (namespace) {
+    case "es":
+      ext = "mjs"
+      break
+    case "ts":
+    case "json":
+      ext = namespace
+      break
+    default:
+      ext = "js"
+  }
+
+  const filename = `${replacePlaceholders(path, { locale })}.${ext}`
+  await writeFile(filename, compiledCatalog)
+  return filename
+}
+
 export function orderByMessage<T extends ExtractedCatalogType>(messages: T): T {
   // hardcoded en-US locale to have consistent sorting
   // @see https://github.com/lingui/js-lingui/pull/1808

packages/cli/src/api/catalog/extractFromFiles.ts

@@ -1,20 +1,21 @@
-import type {
-  ExtractedMessage,
-  ExtractorType,
-  LinguiConfigNormalized,
-} from "@lingui/conf"
+import type { ExtractedMessage, LinguiConfigNormalized } from "@lingui/conf"
 import pico from "picocolors"
 import path from "path"
 import extract from "../extractors"
 import { ExtractedCatalogType, MessageOrigin } from "../types"
 import { prettyOrigin } from "../utils"
+import { Pool, spawn, Worker } from "threads"
+import type { ExtractWorkerFunction } from "../../workers/extractWorker"
 
 function mergePlaceholders(
   prev: Record<string, string[]>,
   next: Record<string, string>
 ) {
   const res = { ...prev }
 
+  // Handle case where next is null or undefined
+  if (!next) return res
+
   Object.entries(next).forEach(([key, value]) => {
     if (!res[key]) {
       res[key] = []
@@ -38,65 +39,109 @@ export async function extractFromFiles(
 
   let catalogSuccess = true
 
-  await Promise.all(
-    paths.map(async (filename) => {
+  if (config.experimental?.multiThread) {
+    catalogSuccess = await extractFromFilesWithWorkers(paths, config, messages)
+  } else {
+    for (const filename of paths) {
       const fileSuccess = await extract(
         filename,
         (next: ExtractedMessage) => {
-          if (!messages[next.id]) {
-            messages[next.id] = {
-              message: next.message,
-              context: next.context,
-              placeholders: {},
-              comments: [],
-              origin: [],
-            }
-          }
-
-          const prev = messages[next.id]
-
-          // there might be a case when filename was not mapped from sourcemaps
-          const filename = next.origin[0]
-            ? path.relative(config.rootDir, next.origin[0]).replace(/\\/g, "/")
-            : ""
-
-          const origin: MessageOrigin = [filename, next.origin[1]]
-
-          if (prev.message && next.message && prev.message !== next.message) {
-            throw new Error(
-              `Encountered different default translations for message ${pico.yellow(
-                next.id
-              )}` +
-                `\n${pico.yellow(prettyOrigin(prev.origin))} ${prev.message}` +
-                `\n${pico.yellow(prettyOrigin([origin]))} ${next.message}`
-            )
-          }
-
-          messages[next.id] = {
-            ...prev,
-            message: prev.message ?? next.message,
-            comments: next.comment
-              ? [...prev.comments, next.comment].sort()
-              : prev.comments,
-            origin: (
-              [...prev.origin, [filename, next.origin[1]]] as MessageOrigin[]
-            ).sort((a, b) => a[0].localeCompare(b[0])),
-            placeholders: mergePlaceholders(
-              prev.placeholders,
-              next.placeholders
-            ),
-          }
+          mergeExtractedMessage(next, messages, config)
         },
-        config,
-        {
-          extractors: config.extractors as ExtractorType[],
-        }
+        config
       )
       catalogSuccess &&= fileSuccess
-    })
-  )
+    }
+  }
 
   if (!catalogSuccess) return undefined
 
   return messages
 }
+
+function mergeExtractedMessage(
+  next: ExtractedMessage,
+  messages: ExtractedCatalogType,
+  config: LinguiConfigNormalized
+) {
+  if (!messages[next.id]) {
+    messages[next.id] = {
+      message: next.message,
+      context: next.context,
+      placeholders: {},
+      comments: [],
+      origin: [],
+    }
+  }
+
+  const prev = messages[next.id]
+
+  // there might be a case when filename was not mapped from sourcemaps
+  const filename = next.origin[0]
+    ? path.relative(config.rootDir, next.origin[0]).replace(/\\/g, "/")
+    : ""
+
+  const origin: MessageOrigin = [filename, next.origin[1]]
+
+  if (prev.message && next.message && prev.message !== next.message) {
+    throw new Error(
+      `Encountered different default translations for message ${pico.yellow(
+        next.id
+      )}` +
+        `\n${pico.yellow(prettyOrigin(prev.origin))} ${prev.message}` +
+        `\n${pico.yellow(prettyOrigin([origin]))} ${next.message}`
+    )
+  }
+
+  messages[next.id] = {
+    ...prev,
+    message: prev.message ?? next.message,
+    comments: next.comment
+      ? [...prev.comments, next.comment].sort()
+      : prev.comments,
+    origin: (
+      [...prev.origin, [filename, next.origin[1]]] as MessageOrigin[]
+    ).sort((a, b) => a[0].localeCompare(b[0])),
+    placeholders: mergePlaceholders(prev.placeholders, next.placeholders),
+  }
+}
+
+async function extractFromFilesWithWorkers(
+  paths: string[],
+  config: LinguiConfigNormalized,
+  messages: ExtractedCatalogType
+): Promise<boolean> {
+  const pool = Pool(() =>
+    spawn<ExtractWorkerFunction>(new Worker("../../workers/extractWorker"))
+  )
+
+  let catalogSuccess = true
+
+  try {
+    if (!config.resolvedConfigPath) {
+      throw new Error(
+        "Multithreading is only supported when lingui config loaded from file system, not passed by API"
+      )
+    }
+
+    paths.map((filename) =>
+      pool.queue(async (worker) => {
+        const result = await worker(filename, config.resolvedConfigPath)
+
+        if (!result.success) {
+          catalogSuccess = false
+        } else {
+          result.messages.forEach((message) => {
+            mergeExtractedMessage(message, messages, config)
+          })
+        }
+      })
+    )
+
+    await pool.completed(true)
+  } finally {
+    await pool.terminate()
+  }
+
+  return catalogSuccess
+}