feat: Implement cache file streaming with progress logging

If cache files get large enough to cause noticeable delay it will be helpful to log progress

Author: FoxxMD

src/backend/common/Cache.ts

@@ -114,7 +114,7 @@ export class MSCache {
             logger.debug(`Building file cache from ${path.join(config.connection, `${ns}.cache`)}`);
 
             try {
-                const [keyvFile] = initFileCache({ ...config, cacheDir: config.connection, cacheId: `${ns}.cache` }, logger);
+                const [keyvFile] = await initFileCache({ ...config, cacheDir: config.connection, cacheId: `${ns}.cache` }, logger);
                 secondaryCache = keyvFile;
             } catch (e) {
                 logger.warn(e);
@@ -177,38 +177,57 @@ export const flatCacheCreate = (opts: FlatCacheOptions) => {
     });
 }
 
-export const flatCacheLoad = (flatCache: FlatCache, logger: MaybeLogger): void => {
+export const flatCacheLoad = async (flatCache: FlatCache, logger: MaybeLogger): Promise<void> => {
 
     const cachePath = path.join(flatCache.cacheDir, flatCache.cacheId);
     try {
         fileOrDirectoryIsWriteable(cachePath);
     } catch (e) {
-        new Error(`Unable to use path for file cache at ${cachePath}`, { cause: e })
+        throw new Error(`Unable to use path for file cache at ${cachePath}`, { cause: e })
     }
 
-    let loadError: Error;
+    const streamPromise = new Promise((resolve, reject) => {
+        flatCache.loadFileStream(cachePath, (progress: number, total: number) => {
+            logger.debug(`Loading ${progress}/${total} chunks...`);
+        }, () => {
+            resolve(true);
+        }, (err: Error) => {
+            reject(err);
+        });
+    });
 
-    const onlySaveError = (e: Error) => {
-        loadError = e;
-    }
-    flatCache.on('error', onlySaveError);
     try {
-        logger.debug('Loading cache from file...');
-        flatCache.load();
-        if (loadError !== undefined) {
-            throw loadError;
-        }
+        await streamPromise;
         logger.debug(`File cache loaded`);
-        flatCache.off('error', onlySaveError);
+        return;
     } catch (e) {
-        throw new Error(`Unable to use file cache at ${cachePath}`, { cause: e });
+        if (null !== e.message.match(/Cache file .+ does not exist/)) {
+            let loadError: Error;
+            try {
+                const onlySaveError = (e: Error) => {
+                    loadError = e;
+                };
+                flatCache.on('error', onlySaveError);
+                flatCache.load();
+                if (loadError !== undefined) {
+                    throw loadError;
+                }
+                flatCache.off('error', onlySaveError);
+                logger.debug(`File cache loaded`);
+                return;
+            } catch (e) {
+                throw new Error(`Unable to use file cache at ${cachePath}`, { cause: e });
+            }
+        } else {
+            throw new Error(`Unable to use file cache at ${cachePath}`, { cause: e });
+        }
     }
 }
 
-export const initFileCache = (opts: FlatCacheOptions = {}, logger: MaybeLogger = new MaybeLogger()): [Keyv | KeyvStoreAdapter | undefined, FlatCache | undefined] => {
+export const initFileCache = async (opts: FlatCacheOptions = {}, logger: MaybeLogger = new MaybeLogger()): Promise<[Keyv | KeyvStoreAdapter | undefined, FlatCache | undefined]> => {
     const flatCache = flatCacheCreate(opts);
     try {
-        flatCacheLoad(flatCache, logger);
+        await flatCacheLoad(flatCache, logger);
         flatCache.on('error', (e) => {
             logger.warn(e);
         });

src/backend/tests/cache/cache.test.ts

@@ -162,27 +162,21 @@ describe('#Caching', function () {
 
                 const root = getRoot();
                 root.upsert({ cache: () => () => new MSCache(loggerTest, { scrobble: { provider: 'file', connection: process.cwd(), persistInterval: 100 } }) });
-                root.items.cache().init();
 
                 const test = new TestScrobbler();
                 await test.initialize();
-                const play = generatePlay();
-                test.queueScrobble(play, 'testSource');
+                const plays = generatePlays(100);
+                test.queueScrobble(plays, 'testSource');
+                const queued = test.queuedScrobbles.map(x => x.play);
                 await sleep(101);
                 const dirContents = await promises.readdir('.');
                 const hasCache = dirContents.some(x => x === 'ms-scrobble.cache');
                 expect(hasCache).is.true;
 
-                // reinit cache
-                root.upsert({ cache: () => () => new MSCache(loggerTest, { scrobble: { provider: 'file', connection: process.cwd(), persistInterval: 100 } }) });
-                const newCache = root.items.cache();
-                expect(newCache.cacheScrobble).to.be.undefined;
-                newCache.init();
-
                 const newTest = new TestScrobbler();
                 await newTest.initialize();
-                expect(newTest.queuedScrobbles.length).to.eq(1);
-                expect(newTest.queuedScrobbles[0].play.data.track).to.eq(play.data.track);
+                expect(newTest.queuedScrobbles.length).to.eq(plays.length);
+                expect(newTest.queuedScrobbles[0].play.data.track).to.eq(queued[0].data.track);
 
             }, { unsafeCleanup: true });