Fix/watchmode (#259)

* remove old config

* scully(watch-mode): add more files

* scully(watchMode): merge with master and fix double build

* add files to test

* fix(perfomrance metrics): reset metrics on rebuild

* fix(remove stray folder): fox

* more test

* add more test

* fix(watch): add fix for route import

* feat(watch-mode): add the watchmode with the new config

* fix

* fix

* feat(cahche geuss): cach the angular routes

Co-authored-by: Sander Elias <SanderElias@users.noreply.github.com>

Author: jorgeucano

blog/page-1.md

@@ -5,6 +5,7 @@ publish date: 2019-11-26
 slug: look at_my-urls Cool
 slugs:
   - 'page-1'
+  - 'a slug'
 description: This is the first demo page in this sample.
 ---
 
@@ -14,6 +15,7 @@ description: This is the first demo page in this sample.
 
 ```typescript
 console.log('hello world');
+console.log('yah');
 ```
 
 Related information [page-2](/blog/page-2)

scully/scully.ts

@@ -8,7 +8,7 @@ import {join} from 'path';
 import * as yargs from 'yargs';
 import './pluginManagement/systemPlugins';
 import {startBackgroundServer} from './startBackgroundServer';
-import {loadConfig} from './utils/config';
+import {loadConfig, scullyConfig} from './utils/config';
 import {moveDistAngular} from './utils/fsAngular';
 import {httpGetJson} from './utils/httpGetJson';
 import {isPortTaken} from './utils/isPortTaken';
@@ -20,16 +20,18 @@ import {bootServe, isBuildThere, watchMode} from './watchMode';
 /** the default of 10 is too shallow for generating pages. */
 require('events').defaultMaxListeners = 100;
 
-let port;
-// tslint:disable-next-line:variable-name
-export let _options = {};
-
 const {argv: options} = yargs.option('port', {
   alias: 'p',
   type: 'number',
   description: 'The port to run on',
 });
 
+const {watch} = yargs
+  .boolean('wm')
+  .default('wm', false)
+  .alias('wm', 'watch')
+  .describe('wm', 'Use this flag for use the watch mode into scully').argv;
+
 const {removeStaticDist} = yargs
   .boolean('RSD')
   .default('RSD', false)
@@ -74,14 +76,14 @@ if (process.argv.includes('version')) {
       logError('Could not connect to server');
       process.exit(15);
     }
-
-    console.log('servers available');
-    await startScully();
-
-    if (process.argv.includes('watch')) {
-      _options = options;
-      watchMode();
+    if (watch) {
+      watchMode(
+        join(scullyConfig.homeFolder, scullyConfig.distFolder) ||
+          join(scullyConfig.homeFolder, './dist/browser')
+      );
     } else {
+      console.log('servers available');
+      await startScully();
       if (!isTaken) {
         // kill serve ports
         await httpGetJson(`http://${scullyConfig.hostName}:${scullyConfig.appPort}/killMe`, {

scully/utils/defaultAction.ts

@@ -9,29 +9,33 @@ import {storeRoutes} from '../systemPlugins/storeRoutes';
 import {writeToFs} from '../systemPlugins/writeToFs.plugin';
 import {asyncPool} from './asyncPool';
 import {chunk} from './chunk';
-import {loadConfig, updateScullyConfig} from './config';
-import {ScullyConfig} from './interfacesandenums';
+import {loadConfig} from './config';
 import {log, logWarn} from './log';
 import {performanceIds} from './performanceIds';
 
-const {baseFilter} = yargs
+export const {baseFilter} = yargs
   .string('bf')
   .alias('bf', 'baseFilter')
   .default('bf', '')
   .describe('bf', 'provide a minimatch glob for the unhandled routes').argv;
 
+const cache = new Set<string>();
+
 console.log(baseFilter);
-export const generateAll = async (config?: Partial<ScullyConfig>, localBaseFilter = baseFilter) => {
-  if (config) {
-    await updateScullyConfig(config);
-  }
+export const generateAll = async (localBaseFilter = baseFilter) => {
   await loadConfig;
   try {
-    log('Finding all routes in application.');
-    performance.mark('startTraverse');
-    const unhandledRoutes = await traverseAppRoutes();
-    performance.mark('stopTraverse');
-    performanceIds.add('Traverse');
+    let unhandledRoutes;
+    if (cache.size == 0) {
+      log('Finding all routes in application.');
+      performance.mark('startTraverse');
+      unhandledRoutes = await traverseAppRoutes();
+      performance.mark('stopTraverse');
+      performanceIds.add('Traverse');
+      unhandledRoutes.forEach(r => cache.add(r));
+    } else {
+      unhandledRoutes = [...cache.keys()];
+    }
 
     if (unhandledRoutes.length < 1) {
       logWarn('No routes found in application, are you sure you installed the router? Terminating.');

scully/utils/fsAngular.ts

@@ -7,6 +7,7 @@ import {restartStaticServer, startScullyWatchMode} from '../watchMode';
 import {scullyConfig} from './config';
 import {createFolderFor} from './createFolderFor';
 import {green, log, logWarn} from './log';
+import {baseFilter} from './defaultAction';
 
 export async function checkChangeAngular(
   folder = join(scullyConfig.homeFolder, scullyConfig.distFolder) ||
@@ -27,7 +28,7 @@ function reWatch(folder, reset = true, watch = false) {
       // TODO test on mac, figure out what's coming in.
       // only act upon changes of the actual folder I'm interested in.
       filter(r => r.fileName.includes(filename)),
-      // tap(r => log(`debug only: filename:"${yellow(filename)}" changedName:"${yellow(r.fileName)}"`)),
+      filter(r => !r.fileName.includes('scully-routes.json')),
       /** give the CLI some time to finnish */
       debounceTime(1000),
       // tap(console.log),
@@ -70,7 +71,7 @@ export async function moveDistAngular(src, dest, {reset = true, removeStaticDist
     if (reset) {
       restartStaticServer();
     } else if (watch) {
-      startScullyWatchMode();
+      startScullyWatchMode(baseFilter);
     }
   } catch (e) {
     /**

scully/utils/fsFolder.ts

@@ -6,6 +6,7 @@ import {log, red} from './log';
 import {watch} from 'chokidar';
 import {scullyConfig} from './config';
 import {startScullyWatchMode} from '../watchMode';
+import * as fs from 'fs';
 
 // tslint:disable-next-line:no-shadowed-variable
 export async function checkStaticFolder() {
@@ -15,15 +16,19 @@ export async function checkStaticFolder() {
     // tslint:disable-next-line:forin
     for (const property in config) {
       if (config[property].type === 'contentFolder') {
-        // @ts-ignore
-        const fileName = config[property].slug.folder.replace('./', '');
-        console.log(fileName);
-        if (!folder.find(f => f === fileName)) {
-          folder.push(fileName);
-          if (existFolder(fileName)) {
-            reWatch(fileName);
-          } else {
-            log(`${red(`${fileName} folder not found`)}.`);
+        // tslint:disable-next-line:forin
+        for (const slug in config[property]) {
+          if (config[property][slug].folder !== undefined) {
+            // @ts-ignore
+            const fileName = config[property].slug.folder.replace('./', '');
+            if (!folder.find(f => f === fileName)) {
+              folder.push(fileName);
+              if (existFolder(fileName)) {
+                reWatch(fileName, property);
+              } else {
+                log(`${red(`${fileName} folder not found`)}.`);
+              }
+            }
           }
         }
       }
@@ -33,33 +38,29 @@ export async function checkStaticFolder() {
   }
 }
 
-function reWatch(folder) {
+function reWatch(folder, url) {
   const filename = join(folder);
   watchFolder(filename)
-    .pipe(
-      // TODO test on mac, figure out what's coming in.
-      // only act upon changes of the actual folder I'm interested in.
-      filter(r => r.fileName.startsWith(folder)),
-      throttleTime(3000)
-    )
+    .pipe(throttleTime(10000))
     .subscribe({
       next: v => {
-        if (v.eventType !== 'addDir') {
-          console.log('--------------------------------------------------');
-          console.log(`New ${v.eventType} in ${v.fileName}, re run scully.`);
-          console.log('--------------------------------------------------');
-          startScullyWatchMode();
-        }
+        console.log('--------------------------------------------------');
+        console.log(`New ${v.eventType} in ${v.fileName}, re run scully.`);
+        console.log('--------------------------------------------------');
+        startScullyWatchMode(url);
       },
     });
 }
 
 function watchFolder(folder): Observable<{eventType: string; fileName: string}> {
+  console.log('--------------------------------------------------');
+  console.log(`Watching ${folder} for change.`);
+  console.log('--------------------------------------------------');
   return new Observable(obs => {
     let watcher;
     try {
-      watcher = watch(folder).on('all', (eventType, fileName) => {
-        obs.next({eventType, fileName});
+      watcher = fs.watch(folder, (event, fname) => {
+        obs.next({eventType: event, fileName: fname});
       });
     } catch (e) {
       obs.error(e);

scully/utils/nodeFsFolder.ts

@@ -0,0 +1,58 @@
+/*
+ * NodeFsFolder
+ * read directorys and files
+ * reactive 100%
+ * */
+import {existsSync} from 'fs';
+import {join} from 'path';
+import {log, red} from './log';
+import {BehaviorSubject, Observable} from 'rxjs';
+import * as fs from 'fs';
+import {throttleTime} from 'rxjs/operators';
+
+// tslint:disable-next-line:no-shadowed-variable
+function readPath(folder: string) {
+  // tslint:disable-next-line:variable-name
+  const _path: string = join(__dirname, folder);
+  if (existFolder(_path)) {
+    return true;
+  } else {
+    log(`${red(`${folder} folder not found, directory ${_path}`)}.`);
+    return false;
+  }
+}
+
+function existFolder(src) {
+  try {
+    return existsSync(src);
+  } catch (e) {
+    return false;
+  }
+}
+
+function watchFolder(folder): Observable<{eventType: string; fileName: string}> {
+  return new Observable(obs => {
+    let watcher;
+    try {
+      watcher = fs.watch(folder, (event, fname) => {
+        console.log('--------------------------------------------------');
+        console.log(`New ${event} in ${fname}, re run scully.`);
+        console.log('--------------------------------------------------');
+        obs.next({eventType: event, fileName: fname});
+      });
+    } catch (e) {
+      obs.error(e);
+    }
+    return () => watcher.close();
+  });
+}
+
+export function startWatch(folder: string) {
+  console.log('start process', folder);
+  if (readPath(folder)) {
+    const fxwatch = this.watchFolder(folder);
+    fxwatch.pipe(throttleTime(3000)).subscribe(x => {
+      console.log(x);
+    });
+  }
+}

scully/utils/startup.ts

@@ -1,28 +1,32 @@
-import {generateAll} from './defaultAction';
-import {ScullyConfig} from './interfacesandenums';
 import {performance, PerformanceObserver, PerformanceObserverCallback} from 'perf_hooks';
+import {generateAll} from './defaultAction';
 import {log, yellow} from './log';
 import {performanceIds} from './performanceIds';
 
 /**
  * Starts the entire process
  * @param config:ScullyConfig
  */
-export const startScully = (config?: Partial<ScullyConfig>) => {
-  let routeCount = 0;
+export const startScully = (url?: string) => {
   return new Promise(resolve => {
     performance.mark('startDuration');
     performanceIds.add('Duration');
     let innerResolve;
     const durationProm = new Promise(r => (innerResolve = r));
     const obs = new PerformanceObserver(measurePerformance(innerResolve));
     obs.observe({entryTypes: ['measure'], buffered: true});
-    const numberOfRoutesProm = generateAll(config).then(routes => {
-      performance.mark('stopDuration');
-      /** measure all performance checks */
-      [...performanceIds.values()].forEach(id => performance.measure(id, `start${id}`, `stop${id}`));
-      return routes.length;
-    });
+    const numberOfRoutesProm = generateAll(url)
+      .then(routes => {
+        performance.mark('stopDuration');
+        /** measure all performance checks */
+        try {
+          [...performanceIds.values()].forEach(id => performance.measure(id, `start${id}`, `stop${id}`));
+        } catch (e) {
+          console.error(e);
+        }
+        return routes.length;
+      })
+      .catch(() => 0);
     Promise.all([numberOfRoutesProm, durationProm]).then(([numberOfRoutes, durations]) =>
       resolve({numberOfRoutes, durations})
     );
@@ -53,6 +57,7 @@ function measurePerformance(resolve: (value?: unknown) => void): PerformanceObse
     // console.log(durations);
     performance.clearMarks();
     observer.disconnect();
+    performanceIds.clear();
     resolve(durations);
   };
 }

scully/watchMode.ts

@@ -1,7 +1,7 @@
 import {existsSync} from 'fs-extra';
 import {join} from 'path';
 import * as yargs from 'yargs';
-import {ScullyConfig, startScully} from '.';
+import {scullyConfig, ScullyConfig, startScully} from '.';
 import {checkChangeAngular} from './utils/fsAngular';
 import {checkStaticFolder} from './utils/fsFolder';
 import {httpGetJson} from './utils/httpGetJson';
@@ -35,12 +35,13 @@ export async function bootServe(scullyConfig: ScullyConfig) {
 }
 
 // TODO : we need rewrite this to observables for don't have memory leaks
-export async function watchMode() {
+// tslint:disable-next-line:variable-name
+export async function watchMode(path: string) {
   await checkStaticFolder();
   // g for generate and the q for quit
   checkForManualRestart();
   // @ts-ignore
-  await checkChangeAngular(_options.path, false, true);
+  await checkChangeAngular(path, false, true);
 }
 
 export function checkForManualRestart() {
@@ -49,21 +50,22 @@ export function checkForManualRestart() {
     output: process.stdout,
   });
 
-  readline.question(`Press g for manual regenerate, or q for close the server. \n`, command => {
+  readline.question(`Press g for manual regenerate, or q for close the server.\n`, command => {
     if (command.toLowerCase() === 'g') {
-      startScully().then(() => checkForManualRestart());
+      startScully().then(() => {
+        readline.close();
+        checkForManualRestart();
+      });
     } else if (command.toLowerCase() === 'q') {
-      readline.close();
       process.exit(0);
     } else {
-      readline.close();
-      checkForManualRestart();
+      console.log(`Press g for manual regenerate, or q for close the server.`);
     }
   });
 }
 
-export function startScullyWatchMode() {
-  startScully();
+export function startScullyWatchMode(url: string) {
+  startScully(url);
 }
 
 function startStaticServer() {