Add support for cloud storage compression

Cloud storage is a limited resource, and thus it
makes sense to support data compression before
sending the data to cloud storage.

A new hidden setting allows to toggle on
cloud storage compression:

name: cloudStorageCompression
default: false

By default, this hidden setting is `false`, and a
user must set it to `true` to enable compression
of cloud storage items.

This hidden setting will eventually be toggled
to `true` by default, when there is good confidence
a majority of users are using a version of uBO
which can properly handle compressed cloud storage
items.

A cursory assessment shows that compressed items
are roughly 40-50% smaller in size.

Author: gorhill

platform/chromium/vapi-background.js

@@ -1540,10 +1540,10 @@ vAPI.cloud = (( ) => {
     // good thing given chrome.storage.sync.MAX_WRITE_OPERATIONS_PER_MINUTE
     // and chrome.storage.sync.MAX_WRITE_OPERATIONS_PER_HOUR.
 
-    const getCoarseChunkCount = async function(dataKey) {
+    const getCoarseChunkCount = async function(datakey) {
         const keys = {};
         for ( let i = 0; i < maxChunkCountPerItem; i += 16 ) {
-            keys[dataKey + i.toString()] = '';
+            keys[datakey + i.toString()] = '';
         }
         let bin;
         try {
@@ -1553,13 +1553,13 @@ vAPI.cloud = (( ) => {
         }
         let chunkCount = 0;
         for ( let i = 0; i < maxChunkCountPerItem; i += 16 ) {
-            if ( bin[dataKey + i.toString()] === '' ) { break; }
+            if ( bin[datakey + i.toString()] === '' ) { break; }
             chunkCount = i + 16;
         }
         return chunkCount;
     };
 
-    const deleteChunks = function(dataKey, start) {
+    const deleteChunks = function(datakey, start) {
         const keys = [];
 
         // No point in deleting more than:
@@ -1570,34 +1570,37 @@ vAPI.cloud = (( ) => {
             Math.ceil(maxStorageSize / maxChunkSize)
         );
         for ( let i = start; i < n; i++ ) {
-            keys.push(dataKey + i.toString());
+            keys.push(datakey + i.toString());
         }
         if ( keys.length !== 0 ) {
             webext.storage.sync.remove(keys);
         }
     };
 
-    const push = async function(dataKey, data) {
-        let bin = {
-            'source': options.deviceName || options.defaultDeviceName,
-            'tstamp': Date.now(),
-            'data': data,
-            'size': 0
+    const push = async function(details) {
+        const { datakey, data, encode } = details;
+        const item = {
+            source: options.deviceName || options.defaultDeviceName,
+            tstamp: Date.now(),
+            data,
         };
-        bin.size = JSON.stringify(bin).length;
-        const item = JSON.stringify(bin);
+        const json = JSON.stringify(item);
+        const encoded = encode instanceof Function
+            ? await encode(json)
+            : json;
 
         // Chunkify taking into account QUOTA_BYTES_PER_ITEM:
         //   https://developer.chrome.com/extensions/storage#property-sync
         //   "The maximum size (in bytes) of each individual item in sync
         //   "storage, as measured by the JSON stringification of its value
         //   "plus its key length."
-        bin = {};
-        let chunkCount = Math.ceil(item.length / maxChunkSize);
+        const bin = {};
+        const chunkCount = Math.ceil(encoded.length / maxChunkSize);
         for ( let i = 0; i < chunkCount; i++ ) {
-            bin[dataKey + i.toString()] = item.substr(i * maxChunkSize, maxChunkSize);
+            bin[datakey + i.toString()]
+                = encoded.substr(i * maxChunkSize, maxChunkSize);
         }
-        bin[dataKey + chunkCount.toString()] = ''; // Sentinel
+        bin[datakey + chunkCount.toString()] = ''; // Sentinel
 
         try {
             await webext.storage.sync.set(bin);
@@ -1606,18 +1609,19 @@ vAPI.cloud = (( ) => {
         }
 
         // Remove potentially unused trailing chunks
-        deleteChunks(dataKey, chunkCount);
+        deleteChunks(datakey, chunkCount);
     };
 
-    const pull = async function(dataKey) {
+    const pull = async function(details) {
+        const { datakey, decode } = details;
 
-        const result = await getCoarseChunkCount(dataKey);
+        const result = await getCoarseChunkCount(datakey);
         if ( typeof result !== 'number' ) {
             return result;
         }
         const chunkKeys = {};
         for ( let i = 0; i < result; i++ ) {
-            chunkKeys[dataKey + i.toString()] = '';
+            chunkKeys[datakey + i.toString()] = '';
         }
 
         let bin;
@@ -1633,31 +1637,35 @@ vAPI.cloud = (( ) => {
         //   happen when the number of chunks is a multiple of
         //   chunkCountPerFetch. Hence why we must also test against
         //   undefined.
-        let json = [], jsonSlice;
+        let encoded = [];
         let i = 0;
         for (;;) {
-            jsonSlice = bin[dataKey + i.toString()];
-            if ( jsonSlice === '' || jsonSlice === undefined ) { break; }
-            json.push(jsonSlice);
+            const slice = bin[datakey + i.toString()];
+            if ( slice === '' || slice === undefined ) { break; }
+            encoded.push(slice);
             i += 1;
         }
+        encoded = encoded.join('');
+        const json = decode instanceof Function
+            ? await decode(encoded)
+            : encoded;
         let entry = null;
         try {
-            entry = JSON.parse(json.join(''));
+            entry = JSON.parse(json);
         } catch(ex) {
         }
         return entry;
     };
 
-    const used = async function(dataKey) {
+    const used = async function(datakey) {
         if ( webext.storage.sync.getBytesInUse instanceof Function === false ) {
             return;
         }
-        const coarseCount = await getCoarseChunkCount(dataKey);
+        const coarseCount = await getCoarseChunkCount(datakey);
         if ( typeof coarseCount !== 'number' ) { return; }
         const keys = [];
         for ( let i = 0; i < coarseCount; i++ ) {
-            keys.push(`${dataKey}${i}`);
+            keys.push(`${datakey}${i}`);
         }
         let results;
         try {

src/js/background.js

@@ -47,6 +47,7 @@ const µBlock = (( ) => { // jshint ignore:line
         cacheStorageAPI: 'unset',
         cacheStorageCompression: true,
         cacheControlForFirefox1376932: 'no-cache, no-store, must-revalidate',
+        cloudStorageCompression: false,
         cnameIgnoreList: 'unset',
         cnameIgnore1stParty: true,
         cnameIgnoreExceptions: true,

src/js/cachestorage.js

@@ -195,22 +195,48 @@
             return dbPromise;
         };
 
+        const fromBlob = function(data) {
+            if ( data instanceof Blob === false ) {
+                return Promise.resolve(data);
+            }
+            return new Promise(resolve => {
+                const blobReader = new FileReader();
+                blobReader.onloadend = ev => {
+                    resolve(new Uint8Array(ev.target.result));
+                };
+                blobReader.readAsArrayBuffer(data);
+            });
+        };
+
+        const toBlob = function(data) {
+            const value = data instanceof Uint8Array
+                ? new Blob([ data ])
+                : data;
+            return Promise.resolve(value);
+        };
+
+        const compress = function(store, key, data) {
+            return µBlock.lz4Codec.encode(data, toBlob).then(value => {
+                store.push({ key, value });
+            });
+        };
+
+        const decompress = function(store, key, data) {
+            return µBlock.lz4Codec.decode(data, fromBlob).then(data => {
+                store[key] = data;
+            });
+        };
+
         const getFromDb = async function(keys, keyvalStore, callback) {
             if ( typeof callback !== 'function' ) { return; }
             if ( keys.length === 0 ) { return callback(keyvalStore); }
             const promises = [];
             const gotOne = function() {
                 if ( typeof this.result !== 'object' ) { return; }
-                keyvalStore[this.result.key] = this.result.value;
-                if ( this.result.value instanceof Blob === false ) { return; }
-                promises.push(
-                    µBlock.lz4Codec.decode(
-                        this.result.key,
-                        this.result.value
-                    ).then(result => {
-                        keyvalStore[result.key] = result.data;
-                    })
-                );
+                const { key, value } = this.result;
+                keyvalStore[key] = value;
+                if ( value instanceof Blob === false ) { return; }
+                promises.push(decompress(keyvalStore, key, value));
             };
             try {
                 const db = await getDb();
@@ -265,16 +291,10 @@
                     });
                     return;
                 }
-                keyvalStore[entry.key] = entry.value;
+                const { key, value } = entry;
+                keyvalStore[key] = value;
                 if ( entry.value instanceof Blob === false ) { return; }
-                promises.push(
-                    µBlock.lz4Codec.decode(
-                        entry.key,
-                        entry.value
-                    ).then(result => {
-                        keyvalStore[result.key] = result.value;
-                    })
-                );
+                promises.push(decompress(keyvalStore, key, value));
             }).catch(reason => {
                 console.info(`cacheStorage.getAllFromDb() failed: ${reason}`);
                 callback();
@@ -297,19 +317,14 @@
             const entries = [];
             const dontCompress =
                 µBlock.hiddenSettings.cacheStorageCompression !== true;
-            const handleEncodingResult = result => {
-                entries.push({ key: result.key, value: result.data });
-            };
             for ( const key of keys ) {
-                const data = keyvalStore[key];
-                const isString = typeof data === 'string';
+                const value = keyvalStore[key];
+                const isString = typeof value === 'string';
                 if ( isString === false || dontCompress ) {
-                    entries.push({ key, value: data });
+                    entries.push({ key, value });
                     continue;
                 }
-                promises.push(
-                    µBlock.lz4Codec.encode(key, data).then(handleEncodingResult)
-                );
+                promises.push(compress(entries, key, value));
             }
             const finish = ( ) => {
                 if ( callback === undefined ) { return; }

src/js/cloud-ui.js

@@ -48,7 +48,7 @@ if ( self.cloud.datakey === '' ) { return; }
 /******************************************************************************/
 
 const fetchStorageUsed = async function() {
-    const elem = widget.querySelector('#cloudCapacity');
+    let elem = widget.querySelector('#cloudCapacity');
     if ( elem.classList.contains('hide') ) { return; }
     const result = await vAPI.messaging.send('cloudWidget', {
         what: 'cloudUsed',
@@ -58,10 +58,16 @@ const fetchStorageUsed = async function() {
         elem.classList.add('hide');
         return;
     }
+    const units = ' ' + vAPI.i18n('genericBytes');
+    elem.title = result.max.toLocaleString() + units;
     const total = (result.total / result.max * 100).toFixed(1);
-    elem.firstElementChild.style.width = `${total}%`;
+    elem = elem.firstElementChild;
+    elem.style.width = `${total}%`;
+    elem.title = result.total.toLocaleString() + units;
     const used = (result.used / result.total * 100).toFixed(1);
-    elem.firstElementChild.firstElementChild.style.width = `${used}%`;
+    elem = elem.firstElementChild;
+    elem.style.width = `${used}%`;
+    elem.title = result.used.toLocaleString() + units;
 };
 
 /******************************************************************************/