From ac91d586b89a9c012fcd0d6f1105163eef529927 Mon Sep 17 00:00:00 2001 From: Kevin Payravi Date: Thu, 8 Feb 2024 02:34:53 -0600 Subject: [PATCH] Compressing wiki settings Browser sync storage has a 8kb limit per item, which we are quickly approaching. Compressing our wiki settings JSONs reduces storage from ~7.3kb to ~2.4kb. --- background.js | 2 +- pages/popup/popup.js | 4 +- pages/settings/settings.js | 72 ++++++++++++++++------------- scripts/common-functions.js | 64 +++++++++++++++++++++++-- scripts/content-banners.js | 26 ++++++----- scripts/content-search-filtering.js | 7 +-- 6 files changed, 121 insertions(+), 54 deletions(-) diff --git a/background.js b/background.js index 6afcb20..7cc2209 100644 --- a/background.js +++ b/background.js @@ -168,7 +168,7 @@ async function main(url, tabId) { if (matchingSite) { // Get user's settings for the wiki - let settings = storage.wikiSettings || {}; + let settings = await commonFunctionDecompressJSON(storage.wikiSettings) || {}; let id = matchingSite['id']; let siteSetting = settings[id] || storage.defaultWikiAction || 'alert'; diff --git a/pages/popup/popup.js b/pages/popup/popup.js index 32539e2..69bb0fb 100644 --- a/pages/popup/popup.js +++ b/pages/popup/popup.js @@ -108,7 +108,7 @@ document.addEventListener('DOMContentLoaded', () => { sites.forEach((site) => { wikiSettings[site.id] = document.options.defaultWikiAction.value; }); - chrome.storage.sync.set({ 'wikiSettings': wikiSettings }); + chrome.storage.sync.set({ 'wikiSettings': await commonFunctionCompressJSON(wikiSettings) }); }); }); document.querySelectorAll('[name="defaultSearchAction"]').forEach((el) => { @@ -120,7 +120,7 @@ document.addEventListener('DOMContentLoaded', () => { sites.forEach((site) => { searchEngineSettings[site.id] = document.options.defaultSearchAction.value; }); - chrome.storage.sync.set({ 'searchEngineSettings': searchEngineSettings }); + chrome.storage.sync.set({ 'searchEngineSettings': await commonFunctionCompressJSON(searchEngineSettings) }); }); }); }); diff --git a/pages/settings/settings.js b/pages/settings/settings.js index 2c85267..10f21d1 100644 --- a/pages/settings/settings.js +++ b/pages/settings/settings.js @@ -45,10 +45,10 @@ async function loadOptions(lang, textFilter = '') { )); chrome.storage.local.get((localStorage) => { - chrome.storage.sync.get((syncStorage) => { + chrome.storage.sync.get(async (syncStorage) => { const storage = { ...syncStorage, ...localStorage }; - let wikiSettings = storage.wikiSettings || {}; - let searchEngineSettings = storage.searchEngineSettings || {}; + let wikiSettings = await commonFunctionDecompressJSON(storage.wikiSettings || {}); + let searchEngineSettings = await commonFunctionDecompressJSON(storage.searchEngineSettings || {}); let defaultWikiAction = storage.defaultWikiAction || null; let defaultSearchAction = storage.defaultSearchAction || null; @@ -189,45 +189,51 @@ async function loadOptions(lang, textFilter = '') { // Add listeners for when user clicks control: inputDisabled.addEventListener('click', (input) => { - chrome.storage.sync.get({ 'wikiSettings': {} }, (response) => { + chrome.storage.sync.get({ 'wikiSettings': {} }, async (response) => { + let wikiSettings = await commonFunctionDecompressJSON(response.wikiSettings); var key = input.target.getAttribute('data-wiki-key'); - response.wikiSettings.set(key, 'disabled'); - chrome.storage.sync.set({ 'wikiSettings': response.wikiSettings }); + wikiSettings.set(key, 'disabled'); + chrome.storage.sync.set({ 'wikiSettings': await commonFunctionCompressJSON(wikiSettings) }); }); }); inputAlert.addEventListener('click', (input) => { - chrome.storage.sync.get({ 'wikiSettings': {} }, (response) => { + chrome.storage.sync.get({ 'wikiSettings': {} }, async (response) => { + let wikiSettings = await commonFunctionDecompressJSON(response.wikiSettings); var key = input.target.getAttribute('data-wiki-key'); - response.wikiSettings.set(key, 'alert'); - chrome.storage.sync.set({ 'wikiSettings': response.wikiSettings }); + wikiSettings.set(key, 'alert'); + chrome.storage.sync.set({ 'wikiSettings': await commonFunctionCompressJSON(wikiSettings) }); }); }); inputRedirect.addEventListener('click', (input) => { - chrome.storage.sync.get({ 'wikiSettings': {} }, (response) => { + chrome.storage.sync.get({ 'wikiSettings': {} }, async (response) => { + let wikiSettings = await commonFunctionDecompressJSON(response.wikiSettings); var key = input.target.getAttribute('data-wiki-key'); - response.wikiSettings.set(key, 'redirect'); - chrome.storage.sync.set({ 'wikiSettings': response.wikiSettings }); + wikiSettings.set(key, 'redirect'); + chrome.storage.sync.set({ 'wikiSettings': await commonFunctionCompressJSON(wikiSettings) }); }); }); inputSearchEngineDisabled.addEventListener('click', (input) => { - chrome.storage.sync.get({ 'searchEngineSettings': {} }, (response) => { + chrome.storage.sync.get({ 'searchEngineSettings': {} }, async (response) => { + let searchEngineSettings = await commonFunctionDecompressJSON(response.searchEngineSettings); var key = input.target.getAttribute('data-wiki-key'); - response.searchEngineSettings.set(key, 'disabled'); - chrome.storage.sync.set({ 'searchEngineSettings': response.searchEngineSettings }); + searchEngineSettings.set(key, 'disabled'); + chrome.storage.sync.set({ 'searchEngineSettings': await commonFunctionCompressJSON(searchEngineSettings) }); }); }); inputSearchEngineReplace.addEventListener('click', (input) => { - chrome.storage.sync.get({ 'searchEngineSettings': {} }, (response) => { + chrome.storage.sync.get({ 'searchEngineSettings': {} }, async (response) => { + let searchEngineSettings = await commonFunctionDecompressJSON(response.searchEngineSettings); var key = input.target.getAttribute('data-wiki-key'); - response.searchEngineSettings.set(key, 'replace'); - chrome.storage.sync.set({ 'searchEngineSettings': response.searchEngineSettings }); + searchEngineSettings.set(key, 'replace'); + chrome.storage.sync.set({ 'searchEngineSettings': await commonFunctionCompressJSON(searchEngineSettings) }); }); }); inputSearchEngineHide.addEventListener('click', (input) => { - chrome.storage.sync.get({ 'searchEngineSettings': {} }, (response) => { + chrome.storage.sync.get({ 'searchEngineSettings': {} }, async (response) => { + let searchEngineSettings = await commonFunctionDecompressJSON(response.searchEngineSettings); var key = input.target.getAttribute('data-wiki-key'); - response.searchEngineSettings.set(key, 'hide'); - chrome.storage.sync.set({ 'searchEngineSettings': response.searchEngineSettings }); + searchEngineSettings.set(key, 'hide'); + chrome.storage.sync.set({ 'searchEngineSettings': await commonFunctionCompressJSON(searchEngineSettings) }); }); }); @@ -315,63 +321,63 @@ async function loadOptions(lang, textFilter = '') { // Add "select all" button event listeners: const setAllRedirect = document.getElementById('setAllRedirect'); - setAllRedirect.addEventListener('click', () => { + setAllRedirect.addEventListener('click', async () => { const toggles = document.querySelectorAll('#toggles input.toggleRedirect'); for (var i = 0; i < toggles.length; i++) { toggles[i].checked = true; wikiSettings.set(toggles[i].getAttribute('data-wiki-key'), 'redirect'); } - chrome.storage.sync.set({ 'wikiSettings': wikiSettings }); + chrome.storage.sync.set({ 'wikiSettings': await commonFunctionCompressJSON(wikiSettings) }); }); const setAllAlert = document.getElementById('setAllAlert'); - setAllAlert.addEventListener('click', () => { + setAllAlert.addEventListener('click', async () => { const toggles = document.querySelectorAll('#toggles input.toggleAlert'); for (var i = 0; i < toggles.length; i++) { toggles[i].checked = true; wikiSettings.set(toggles[i].getAttribute('data-wiki-key'), 'alert'); } - chrome.storage.sync.set({ 'wikiSettings': wikiSettings }); + chrome.storage.sync.set({ 'wikiSettings': await commonFunctionCompressJSON(wikiSettings) }); }); const setAllDisabled = document.getElementById('setAllDisabled'); - setAllDisabled.addEventListener('click', () => { + setAllDisabled.addEventListener('click', async () => { const toggles = document.querySelectorAll('#toggles input.toggleDisable'); for (var i = 0; i < toggles.length; i++) { toggles[i].checked = true; wikiSettings.set(toggles[i].getAttribute('data-wiki-key'), 'disabled'); } - chrome.storage.sync.set({ 'wikiSettings': wikiSettings }); + chrome.storage.sync.set({ 'wikiSettings': await commonFunctionCompressJSON(wikiSettings) }); }); const setAllSearchEngineDisabled = document.getElementById('setAllSearchEngineDisabled'); - setAllSearchEngineDisabled.addEventListener('click', () => { + setAllSearchEngineDisabled.addEventListener('click', async () => { const toggles = document.querySelectorAll('#toggles input.toggleSearchEngineDisabled'); for (var i = 0; i < toggles.length; i++) { toggles[i].checked = true; searchEngineSettings.set(toggles[i].getAttribute('data-wiki-key'), 'disabled'); } - chrome.storage.sync.set({ 'searchEngineSettings': searchEngineSettings }); + chrome.storage.sync.set({ 'searchEngineSettings': await commonFunctionCompressJSON(searchEngineSettings) }); }); const setAllSearchEngineHide = document.getElementById('setAllSearchEngineHide'); - setAllSearchEngineHide.addEventListener('click', () => { + setAllSearchEngineHide.addEventListener('click', async () => { const toggles = document.querySelectorAll('#toggles input.toggleSearchEngineHide'); for (var i = 0; i < toggles.length; i++) { toggles[i].checked = true; searchEngineSettings.set(toggles[i].getAttribute('data-wiki-key'), 'hide'); } - chrome.storage.sync.set({ 'searchEngineSettings': searchEngineSettings }); + chrome.storage.sync.set({ 'searchEngineSettings': await commonFunctionCompressJSON(searchEngineSettings) }); }); const setAllSearchEngineReplace = document.getElementById('setAllSearchEngineReplace'); - setAllSearchEngineReplace.addEventListener('click', () => { + setAllSearchEngineReplace.addEventListener('click', async () => { const toggles = document.querySelectorAll('#toggles input.toggleSearchEngineReplace'); for (var i = 0; i < toggles.length; i++) { toggles[i].checked = true; searchEngineSettings.set(toggles[i].getAttribute('data-wiki-key'), 'replace'); } - chrome.storage.sync.set({ 'searchEngineSettings': searchEngineSettings }); + chrome.storage.sync.set({ 'searchEngineSettings': await commonFunctionCompressJSON(searchEngineSettings) }); }); }); }); diff --git a/scripts/common-functions.js b/scripts/common-functions.js index 00b0a67..a89b718 100644 --- a/scripts/common-functions.js +++ b/scripts/common-functions.js @@ -1,4 +1,60 @@ var LANGS = ["DE", "EN", "ES", "FI", "FR", "IT", "KO", "PL", "PT", "RU", "TOK", "UK", "ZH"]; +const BASE64REGEX = /^([0-9a-zA-Z+/]{4})*(([0-9a-zA-Z+/]{2}==)|([0-9a-zA-Z+/]{3}=))?$/; + +function b64decode(str) { + const binary_string = atob(str); + const len = binary_string.length; + const bytes = new Uint8Array(new ArrayBuffer(len)); + for (let i = 0; i < len; i++) { + bytes[i] = binary_string.charCodeAt(i); + } + return bytes; +} + +async function commonFunctionDecompressJSON(value) { + // Check if value is base64 encoded: + if (BASE64REGEX.test(value)) { + // Decode into blob + const stream = new Blob([b64decode(value)], { + type: "application/json", + }).stream(); + + // Decompress value + const decompressedReadableStream = stream.pipeThrough( + new DecompressionStream("gzip") + ); + + const resp = new Response(decompressedReadableStream); + const blob = await resp.blob(); + const blobText = JSON.parse(await blob.text()); + return blobText; + } else { + return value; + } +} + +async function commonFunctionCompressJSON(value) { + const stream = new Blob([JSON.stringify(value)], { + type: 'application/json', + }).stream(); + + // Compress stream with gzip + const compressedReadableStream = stream.pipeThrough( + new CompressionStream("gzip") + ); + const compressedResponse = new Response(compressedReadableStream); + + // Convert response to blob and buffer + const blob = await compressedResponse.blob(); + const buffer = await blob.arrayBuffer(); + + // Encode and return string + return btoa( + String.fromCharCode( + ...new Uint8Array(buffer) + ) + ); +} // Load wiki data objects, with each destination having its own object async function commonFunctionGetSiteDataByDestination() { @@ -169,8 +225,8 @@ async function commonFunctionMigrateToV3() { // Migrate wiki settings to new searchEngineSettings and wikiSettings objects sites = await commonFunctionGetSiteDataByOrigin(); let siteSettings = storage.siteSettings || {}; - let searchEngineSettings = storage.searchEngineSettings || {}; - let wikiSettings = storage.wikiSettings || {}; + let searchEngineSettings = await commonFunctionDecompressJSON(storage.searchEngineSettings || {}); + let wikiSettings = await commonFunctionDecompressJSON(storage.wikiSettings) || {}; sites.forEach((site) => { if (!searchEngineSettings[site.id]) { @@ -190,8 +246,8 @@ async function commonFunctionMigrateToV3() { } }); - chrome.storage.sync.set({ 'searchEngineSettings': searchEngineSettings }); - chrome.storage.sync.set({ 'wikiSettings': wikiSettings }); + chrome.storage.sync.set({ 'searchEngineSettings': await commonFunctionCompressJSON(searchEngineSettings) }); + chrome.storage.sync.set({ 'wikiSettings': await commonFunctionCompressJSON(wikiSettings) }); // Remove old object: chrome.storage.sync.remove('siteSettings'); diff --git a/scripts/content-banners.js b/scripts/content-banners.js index 21d9b9e..09ff8e4 100644 --- a/scripts/content-banners.js +++ b/scripts/content-banners.js @@ -207,9 +207,10 @@ function displayRedirectBanner(newUrl, id, destinationName, destinationLanguage, bannerRestoreLink.textContent = '⎌ Restore banner'; bannerControls.appendChild(bannerRestoreLink); bannerRestoreLink.onclick = function (e) { - chrome.storage.sync.get({ 'wikiSettings': {} }, (response) => { - response.wikiSettings.set(id, 'alert'); - chrome.storage.sync.set({ 'wikiSettings': response.wikiSettings }); + chrome.storage.sync.get({ 'wikiSettings': {} }, async (response) => { + let wikiSettings = await commonFunctionDecompressJSON(response.wikiSettings); + wikiSettings.set(id, 'alert'); + chrome.storage.sync.set({ 'wikiSettings': await commonFunctionCompressJSON(wikiSettings) }); e.target.textContent = '✓ Banner restored'; e.target.classList.add('indie-wiki-banner-disabled'); bannerRestoreLink.querySelector('.indie-wiki-banner-redirect').textContent = '↪ Auto redirect this wiki'; @@ -227,9 +228,10 @@ function displayRedirectBanner(newUrl, id, destinationName, destinationLanguage, bannerDisableLink.textContent = '✕ Disable banner for this wiki'; bannerControls.appendChild(bannerDisableLink); bannerDisableLink.onclick = function (e) { - chrome.storage.sync.get({ 'wikiSettings': {} }, (response) => { - response.wikiSettings.set(id, 'disabled'); - chrome.storage.sync.set({ 'wikiSettings': response.wikiSettings }); + chrome.storage.sync.get({ 'wikiSettings': {} }, async (response) => { + let wikiSettings = await commonFunctionDecompressJSON(response.wikiSettings); + wikiSettings.set(id, 'disabled'); + chrome.storage.sync.set({ 'wikiSettings': await commonFunctionCompressJSON(wikiSettings) }); e.target.textContent = '✓ Banner disabled'; e.target.classList.add('indie-wiki-banner-disabled'); bannerDisableLink.querySelector('.indie-wiki-banner-redirect').textContent = '↪ Auto redirect this wiki'; @@ -248,9 +250,10 @@ function displayRedirectBanner(newUrl, id, destinationName, destinationLanguage, bannerRedirectLink.textContent = '↪ Auto redirect this wiki'; bannerControls.appendChild(bannerRedirectLink); bannerRedirectLink.onclick = function (e) { - chrome.storage.sync.get({ 'wikiSettings': {} }, (response) => { - response.wikiSettings.set(id, 'redirect'); - chrome.storage.sync.set({ 'wikiSettings': response.wikiSettings }); + chrome.storage.sync.get({ 'wikiSettings': {} }, async (response) => { + let wikiSettings = await commonFunctionDecompressJSON(response.wikiSettings); + wikiSettings.set(id, 'redirect'); + chrome.storage.sync.set({ 'wikiSettings': await commonFunctionCompressJSON(wikiSettings) }); e.target.textContent = '✓ Redirect enabled'; e.target.classList.add('indie-wiki-banner-disabled'); bannerRedirectLink.querySelector('.indie-wiki-banner-disable').textContent = '✕ Disable banner for this wiki'; @@ -351,8 +354,9 @@ function main() { // Get user's settings for the wiki let id = matchingSite['id']; let siteSetting = 'alert'; - if (storage.wikiSettings && storage.wikiSettings[id]) { - siteSetting = storage.wikiSettings[id]; + let wikiSettings = await commonFunctionDecompressJSON(storage.wikiSettings || {}); + if (wikiSettings[id]) { + siteSetting = wikiSettings[id]; } else if (storage.defaultWikiAction) { siteSetting = storage.defaultWikiAction; } diff --git a/scripts/content-search-filtering.js b/scripts/content-search-filtering.js index 4166d94..2682fdc 100644 --- a/scripts/content-search-filtering.js +++ b/scripts/content-search-filtering.js @@ -13,7 +13,7 @@ Object.prototype.set = function (prop, value) { function base64Decode(text) { text = text.replace(/\s+/g, '').replace(/\-/g, '+').replace(/\_/g, '/'); - return decodeURIComponent(Array.prototype.map.call(window.atob(text), function (c) { return '%' + ('00' + c.charCodeAt(0).toString(16)).slice(-2); }).join('')); + return decodeURIComponent(Array.prototype.map.call(atob(text), function (c) { return '%' + ('00' + c.charCodeAt(0).toString(16)).slice(-2); }).join('')); } // Function to create an observer to watch for mutations on search pages @@ -413,8 +413,9 @@ async function filterSearchResults(searchResults, searchEngine, storage) { // Get user's settings for the wiki let id = matchingSite['id']; let searchFilterSetting = 'replace'; - if (storage.searchEngineSettings && storage.searchEngineSettings[id]) { - searchFilterSetting = storage.searchEngineSettings[id]; + let searchEngineSettings = await commonFunctionDecompressJSON(storage.searchEngineSettings || {}); + if (searchEngineSettings[id]) { + searchFilterSetting = searchEngineSettings[id]; } else if (storage.defaultSearchAction) { searchFilterSetting = storage.defaultSearchAction; }