refactor(scully): move plugins to registery

libs/scully/src/index.ts

@@ -26,7 +26,7 @@ import { startScully } from './lib/utils/startup';
 import { staticServer } from './lib/utils/serverstuff/staticServer';
 import { handleTravesal } from './lib/utils/handlers/handleTravesal';
 import { routeDiscovery } from './lib/utils/handlers/routeDiscovery';
-import { executePluginsForRoute } from './lib/renderPlugins/executePlugins';
+import { renderRoute } from './lib/renderPlugins/executePlugins';
 import { WriteToStorage } from './lib/systemPlugins/writeToFs.plugin';
 import { prod } from './lib/utils/cli-options';
 export * from './lib/utils/log';
@@ -53,7 +53,7 @@ export {
   handleTravesal,
   routeDiscovery,
   WriteToStorage,
-  executePluginsForRoute,
+  renderRoute,
   /** end WIP */
   getConfig as getMyConfig,
   setConfig as setMyConfig,

libs/scully/src/lib/pluginManagement/index.ts

@@ -1,2 +1,2 @@
-export * from './pluginRepository';
-export * from './systemPlugins';
+export { registerPlugin, scullySystem } from './pluginRepository';
+export { findPlugin } from './pluginConfig';

libs/scully/src/lib/pluginManagement/pluginWrap.ts

@@ -1,7 +1,7 @@
 /* eslint-disable no-fallthrough */
 import { performance } from 'perf_hooks';
 import { pluginsError } from '../utils/cli-options';
-import { logError, yellow } from '../utils/log';
+import { logError, yellow, logWrite } from '../utils/log';
 import { performanceIds } from '../utils/performanceIds';
 import { backupData, routeConfigData } from './pluginConfig';
 import { configData, FilePlugin } from './pluginRepository';
@@ -54,7 +54,7 @@ export async function wrap(
  while trying to render route "${yellow(currentRoute || 'unknown')}"
  ${pluginsError ? 'Scully will exit' : 'Results are ignored.'}`
     );
-    console.error(e);
+    logWrite(e);
     if (pluginsError) {
       process.exit(15);
     }

libs/scully/src/lib/renderPlugins/executePlugins.ts

@@ -1,15 +1,21 @@
-import { scullyConfig } from '../utils/config';
-import { plugins } from '../pluginManagement/pluginRepository';
+import {
+  plugins,
+  registerPlugin,
+  scullySystem,
+} from '../pluginManagement/pluginRepository';
 import { HandledRoute } from '../routerPlugins/addOptionalRoutesPlugin';
+import { scullyConfig } from '../utils/config';
 import { logError, yellow } from '../utils/log';
 import { puppeteerRender } from './puppeteerRenderPlugin';
-import { pluginsError } from '../utils/cli-options';
+import { findPlugin } from '../pluginManagement/pluginConfig';
 
-export const executePluginsForRoute = async (route: HandledRoute) => {
+export const renderRoute = Symbol('renderRoute');
+
+const executePluginsForRoute = async (route: HandledRoute) => {
   /** make one array with all handlers for this route, filter out empty ones */
   const handlers = [
     route.type,
-    ...(route.postRenderers || scullyConfig.defaultPostRenderers)
+    ...(route.postRenderers || scullyConfig.defaultPostRenderers),
   ].filter(Boolean);
   const preRender = route.config && route.config.preRenderer;
   if (preRender) {
@@ -33,7 +39,7 @@ export const executePluginsForRoute = async (route: HandledRoute) => {
       return '';
     }
   }
-  const InitialPromise = puppeteerRender(route);
+  const InitialPromise = findPlugin(puppeteerRender)(route);
   return handlers.reduce(async (updatedHTML, plugin) => {
     const html = await updatedHTML;
     const handler = plugins.render[plugin];
@@ -53,3 +59,5 @@ export const executePluginsForRoute = async (route: HandledRoute) => {
     return html;
   }, InitialPromise);
 };
+
+registerPlugin(scullySystem, renderRoute, executePluginsForRoute);

libs/scully/src/lib/renderPlugins/puppeteerRenderPlugin.ts

@@ -13,6 +13,7 @@ import { scullyConfig } from '../utils/config';
 import { logError, yellow, logWarn } from '../utils/log';
 import { launchedBrowser, reLaunch } from './launchedBrowser';
 import { title404 } from '../utils/serverstuff/title404';
+import { registerPlugin, scullySystem } from '../pluginManagement';
 
 const errorredPages = new Map<string, number>();
 
@@ -26,7 +27,9 @@ try {
   // version = jsonc.parse(readFileSync(join(__dirname, '../../../package.json')).toString()).version || '0.0.0';
 }
 
-export const puppeteerRender = async (route: HandledRoute): Promise<string> => {
+export const puppeteerRender = Symbol('puppeteerRender');
+
+const plugin = async (route: HandledRoute): Promise<string> => {
   const timeOutValueInSeconds = 25;
   const pageLoaded = new Subject<void>();
   const path = scullyConfig.hostUrl
@@ -224,7 +227,7 @@ export const puppeteerRender = async (route: HandledRoute): Promise<string> => {
       /** give it a couple of secs */
       await waitForIt(3 * 1000);
       /** retry! */
-      return puppeteerRender(route);
+      return plugin(route);
     }
   }
 
@@ -243,3 +246,5 @@ const windowSet = (page: Page, name: string, value: Serializable) =>
       }
     })
   `);
+
+registerPlugin(scullySystem, puppeteerRender, plugin);

libs/scully/src/lib/routerPlugins/traverseAppRoutesPlugin.ts

@@ -6,10 +6,14 @@ import { scullyConfig } from '../utils/config';
 import { existFolder } from '../utils/fsFolder';
 import { green, log, logError, logWarn, yellow } from '../utils/log';
 import { createFolderFor } from '../utils/createFolderFor';
+import {
+  scullySystem,
+  registerPlugin,
+} from '../pluginManagement/pluginRepository';
 
-export const traverseAppRoutes = async (
-  forceScan = scanRoutes
-): Promise<string[]> => {
+export const traverseAppRoutes = Symbol('traverseAppRoutes');
+
+const plugin = async (forceScan = scanRoutes): Promise<string[]> => {
   const appRootFolder = scullyConfig.projectRoot;
   const routesPath = join(
     scullyConfig.homeFolder,
@@ -152,3 +156,5 @@ export async function addExtraRoutes(): Promise<string[]> {
   }
   return result;
 }
+
+registerPlugin(scullySystem, traverseAppRoutes, plugin);

libs/scully/src/lib/systemPlugins/storeRoutes.ts

@@ -5,10 +5,13 @@ import { watch } from '../utils/cli-options';
 import { scullyConfig } from '../utils/config';
 import { createFolderFor } from '../utils/createFolderFor';
 import { log, logError, logWarn, yellow } from '../utils/log';
+import { registerPlugin, scullySystem } from '../pluginManagement';
 
 export const routesFileName = '/assets/scully-routes.json';
 
-export async function storeRoutes(routes: HandledRoute[]) {
+export const storeRoutes = Symbol('storeRoutes');
+registerPlugin(scullySystem, storeRoutes, storeRoutesPlugin);
+async function storeRoutesPlugin(routes: HandledRoute[]) {
   const files = [
     /** in the scully outfolder */
     join(scullyConfig.outDir, routesFileName),

libs/scully/src/lib/utils/asyncPool.ts

@@ -1,34 +1,41 @@
-import { logError, logWarn, orange, yellow, printProgress } from './log';
 import { waitForIt } from '../renderPlugins/puppeteerRenderPlugin';
+import { logWarn, printProgress } from './log';
+import { performance } from 'perf_hooks';
 
+const progressTime = 100;
 /**
  * takes an array, and runs **MaxParralellTasks** in parralell until all tasks are node
  * @param MaxParralellTasks
  * @param array
  * @param taskFn
  */
-const timeoutVal = 2 * 60 * 1000;
 export async function asyncPool<T>(
   MaxParralellTasks: number,
   array: T[],
   taskFn: (x: T) => Promise<T>
 ): Promise<T[]> {
   const ret = [];
   const executing = [];
+  let logTime = performance.now();
   for (const item of array) {
     const p = taskFn(item);
     ret.push(p);
     const e = p.then(() => executing.splice(executing.indexOf(e), 1));
     e['item'] = item;
     executing.push(e);
+    const now = performance.now();
+    if (now - logTime > progressTime) {
+      printProgress(Math.max(array.length - ret.length, executing.length));
+      logTime = now;
+    }
     if (executing.length >= MaxParralellTasks) {
       await Promise.race(executing);
     }
   }
   while (executing.length > 0) {
     /** inform used tasks are still running. */
     printProgress(executing.length);
-    await Promise.race([...executing, waitForIt(100)]);
+    await Promise.race([...executing, waitForIt(progressTime)]);
   }
   return Promise.all(ret);
 }

libs/scully/src/lib/utils/cli-options.ts

@@ -3,30 +3,31 @@ import yargs from 'yargs';
 export const {
   // eslint-disable-next-line @typescript-eslint/ban-ts-comment
   // @ts-ignore
-  watch,
-  removeStaticDist,
-  openNavigator,
-  ssl,
-  sslCert,
-  sslKey,
-  tds,
-  proxyConfigFile,
+  baseFilter,
+  configFileName,
+  folder,
+  handle404,
   hostName,
-  showBrowser,
+  noLog,
+  openNavigator,
   path,
+  pjFirst,
+  pluginsError,
   port,
-  folder,
-  sge,
-  configFileName,
+  prod,
   project,
-  baseFilter,
+  proxyConfigFile,
+  removeStaticDist,
   routeFilter,
   scanRoutes,
-  pjFirst,
   serverTimeout,
-  pluginsError,
-  handle404,
-  prod,
+  sge,
+  showBrowser,
+  ssl,
+  sslCert,
+  sslKey,
+  tds,
+  watch,
 } =
   /** return the argv */
   // eslint-disable-next-line @typescript-eslint/ban-ts-comment
@@ -111,6 +112,11 @@ export const {
       'cf',
       'provide name of the config file to use. if the option --project is also there that takes precedence)'
     )
+    /** don't log info lines to console */
+    .boolean('nl')
+    .default('nl', false)
+    .alias('nl', 'noLog')
+    .describe('nl', 'Log warnings and errors only')
     /** projectName */
     .string('pr')
     .alias('pr', 'project')

libs/scully/src/lib/utils/handlers/defaultAction.ts

@@ -1,3 +1,8 @@
+import {
+  findPlugin,
+  registerPlugin,
+  scullySystem,
+} from '../../pluginManagement';
 import { launchedBrowser } from '../../renderPlugins/launchedBrowser';
 import { HandledRoute } from '../../routerPlugins/addOptionalRoutesPlugin';
 import { baseFilter } from '../cli-options';
@@ -9,12 +14,13 @@ import { handleTravesal } from './handleTravesal';
 import { renderParallel } from './renderParallel';
 import { routeDiscovery } from './routeDiscovery';
 
-export const generateAll = async (
-  localBaseFilter = baseFilter
-): Promise<HandledRoute[]> => {
+export const generateAll = Symbol('generateAll');
+registerPlugin(scullySystem, generateAll, plugin);
+
+async function plugin(localBaseFilter = baseFilter): Promise<HandledRoute[]> {
   await loadConfig;
   try {
-    const unhandledRoutes = await handleTravesal();
+    const unhandledRoutes = await findPlugin(handleTravesal)();
 
     const handledRoutes = await routeDiscovery(
       unhandledRoutes,
@@ -24,7 +30,7 @@ export const generateAll = async (
     const discoveryDone = handleRouteDiscoveryDone(handledRoutes);
 
     /** launch the browser, its shared among renderers */
-    const browser = await launchedBrowser();
+    await launchedBrowser();
     /** start handling each route, works in chunked parallel mode  */
     await renderParallel(handledRoutes);
     /** wait for routeDiscoveryDone plugins to be ready. they can still be running. */
@@ -37,4 +43,4 @@ export const generateAll = async (
     log(e);
   }
   return [];
-};
+}

libs/scully/src/lib/utils/handlers/handleTravesal.ts

@@ -1,20 +1,23 @@
 import { performance } from 'perf_hooks';
+import { findPlugin } from '../../pluginManagement/pluginConfig';
 import { traverseAppRoutes } from '../../routerPlugins/traverseAppRoutesPlugin';
-import { rawRoutesCache, flushRawRoutesCache } from '../cache';
+import { flushRawRoutesCache, rawRoutesCache } from '../cache';
 import { log, logWarn } from '../log';
 import { performanceIds } from '../performanceIds';
+import { registerPlugin, scullySystem } from '../../pluginManagement';
 
-export async function handleTravesal(
-  { forceScan } = { forceScan: false }
-): Promise<string[]> {
+export const handleTravesal = Symbol('handleTravesal');
+registerPlugin(scullySystem, handleTravesal, plugin);
+
+async function plugin({ forceScan } = { forceScan: false }): Promise<string[]> {
   let unhandledRoutes: string[];
   if (forceScan) {
     flushRawRoutesCache();
   }
   if (rawRoutesCache.size === 0) {
     log('Finding all routes in application.');
     performance.mark('startTraverse');
-    unhandledRoutes = await traverseAppRoutes();
+    unhandledRoutes = await findPlugin(traverseAppRoutes)();
     performance.mark('stopTraverse');
     performanceIds.add('Traverse');
     unhandledRoutes.forEach((r) => rawRoutesCache.add(r));

libs/scully/src/lib/utils/handlers/renderParallel.ts

@@ -1,21 +1,22 @@
 import { performance } from 'perf_hooks';
 import { findPlugin } from '../../pluginManagement/pluginConfig';
-import { executePluginsForRoute } from '../../renderPlugins/executePlugins';
+import { renderRoute } from '../../renderPlugins/executePlugins';
+import { waitForIt } from '../../renderPlugins/puppeteerRenderPlugin';
 import { WriteToStorage } from '../../systemPlugins/writeToFs.plugin';
 import { asyncPool } from '../asyncPool';
 import { scullyConfig } from '../config';
-import { logError, logWarn } from '../log';
+import { logWarn } from '../log';
 import { performanceIds } from '../performanceIds';
-import { waitForIt } from '../../renderPlugins/puppeteerRenderPlugin';
 
 const writeToFs = findPlugin(WriteToStorage);
 
 const reThrow = (e) => {
   throw new Error(e);
 };
+const executePluginsForRoute = findPlugin(renderRoute);
 
 export async function renderParallel(dataRoutes: any[]): Promise<any[]> {
-  const renderRoute = (route, tries = 0) =>
+  const routeRender = (route, tries = 0) =>
     Promise.race([
       executePluginsForRoute(route),
       /** sometimes puppeteer just dies without error or completing, this will kill the render after 1.5 minute (takes in account that some pages are _slow_) */
@@ -28,7 +29,7 @@ export async function renderParallel(dataRoutes: any[]): Promise<any[]> {
           /** don't log on first error, puppeteer is flakey, just retry without notifying dev  */
           logWarn(`  route: ${route.route}. Try ${tries} failed with ${e}`);
         }
-        return tries < 3 ? renderRoute(route, tries + 1) : reThrow(e);
+        return tries < 3 ? routeRender(route, tries + 1) : reThrow(e);
       })
       .then((html: string) => html && writeToFs(route.route, html));
   performance.mark('startRender');
@@ -38,7 +39,7 @@ export async function renderParallel(dataRoutes: any[]): Promise<any[]> {
     renderPool = await asyncPool(
       scullyConfig.maxRenderThreads,
       dataRoutes,
-      renderRoute
+      routeRender
     );
   } catch (e) {
     console.log('oops during rendering?', e);

libs/scully/src/lib/utils/handlers/routeDiscovery.ts

@@ -7,6 +7,7 @@ import { storeRoutes } from '../../systemPlugins/storeRoutes';
 import { log, logError } from '../log';
 import { performanceIds } from '../performanceIds';
 import { routeFilter } from '../cli-options';
+import { findPlugin } from '../../pluginManagement';
 
 export async function routeDiscovery(
   unhandledRoutes: string[],
@@ -45,7 +46,7 @@ export async function routeDiscovery(
   /** save routerinfo, so its available during rendering */
   if (localBaseFilter === '' && routeFilter === '') {
     /** only store when the routes are complete  */
-    await storeRoutes(handledRoutes);
+    await findPlugin(storeRoutes)(handledRoutes);
   }
   return handledRoutes;
 }