WIP: safe loading of config.json (ref. #48)

merge-requests/23/merge
Michał 'rysiek' Woźniak 2024-02-01 01:11:14 +00:00
rodzic 2409e716ef
commit 5bca087442
2 zmienionych plików z 271 dodań i 189 usunięć

Wyświetl plik

@ -118,8 +118,12 @@ beforeAll(async ()=>{
* mocking caches.match()
* https://developer.mozilla.org/en-US/docs/Web/API/CacheStorage/match#browser_compatibility
*/
caches.match = async (url) => {
let cache = await caches.open('v1')
caches.match = async (url, options={}) => {
let cn = 'v1'
if ('cacheName' in options) {
cn = options.cacheName
}
let cache = await caches.open(cn)
return cache.match(url)
}
@ -210,14 +214,14 @@ beforeAll(async ()=>{
}
// wait for caching of a URL, looped up to `tries` times
window.waitForCacheAction = (url, action="add", tries=100) => {
window.waitForCacheAction = (url, cache_name, action="add", tries=100) => {
if (action != "add" && action != "remove") {
throw new Error('waitForCacheAction()\'s action parameter can only be "add" or "remove".')
}
console.log('*** WAITING FOR CACHE ACTION:', action, '\n - url:', url)
console.log(`*** WAITING FOR CACHE (${cache_name}) ACTION:`, action, '\n - url:', url)
return new Promise(async (resolve, reject)=>{
// get the cache object
let cache = await caches.open('v1')
let cache = await caches.open(cache_name)
// try to match until we succeed, or run out of tries
for (let i=0; i<tries; i++) {
// search the URL
@ -290,6 +294,13 @@ beforeEach(async ()=>{
await caches.delete('v1')
}
})
await caches
.has('v1:temp')
.then(async (hasv1) => {
if (hasv1) {
await caches.delete('v1:temp')
}
})
// make sure we're starting with a clean slate in LibResilientPluginConstructors
window.LibResilientPluginConstructors = new Map()
// keeping track of whether the SW got installed
@ -501,7 +512,7 @@ describe('service-worker', async () => {
// if we don't make sure that the caching has completed, we will get an error.
// so we wait until config.json is cached, and use that to verify it is in fact cached
assertEquals(
await window.waitForCacheAction(window.location.origin + 'config.json'),
await window.waitForCacheAction(window.location.origin + 'config.json', 'v1'),
mock_response_data.data
);
})
@ -753,17 +764,17 @@ describe('service-worker', async () => {
await window.getMockedResponse(config_url, {}, mock_response_data)
)
})
// service worker is a go!
await import("../../service-worker.js?" + window.test_id);
await self.dispatchEvent(new Event('install'))
await self.waitForSWInstall()
assertSpyCalls(self.fetch, 0)
assertEquals(typeof self.LibResilientConfig, 'object')
assertEquals(self.LibResilientConfig.defaultPluginTimeout, 5000)
assertEquals(self.LibResilientConfig.plugins, [{name: "cache"}])
assertEquals(self.LibResilientConfig.loggedComponents, ['service-worker', 'cache'])
assertSpyCalls(self.fetch, 0)
})
it("should use a stale cached valid config.json file without a fetch, then retrieve and cache a fresh config.json using the configured plugins", async () => {
@ -836,7 +847,7 @@ describe('service-worker', async () => {
// so as not to end up in a forever loop
for (let i=0; i<100; i++) {
// did we get the new config?
if (await window.waitForCacheAction(config_url) === mock_response_data2.data) {
if (await window.waitForCacheAction(config_url, 'v1:temp') === mock_response_data2.data) {
// we did! we're done
return true;
}
@ -909,9 +920,9 @@ describe('service-worker', async () => {
// waiting for potential caching of the "new" config
for (let i=0; i<100; i++) {
// did we get the new config?
if (await window.waitForCacheAction(config_url) === mock_response_data2.data) {
if (await window.waitForCacheAction(config_url, 'v1') === mock_response_data2.data) {
// we did! that's a paddling!
throw new Error('New config failed to cache, apparently!')
throw new Error('New config should not have been cached!')
}
}
})
@ -981,7 +992,7 @@ describe('service-worker', async () => {
// waiting for potential caching of the "new" config
for (let i=0; i<100; i++) {
// did we get the new config?
if (await window.waitForCacheAction(config_url) === mock_response_data2.data) {
if (await window.waitForCacheAction(config_url, 'v1') === mock_response_data2.data) {
// we did! that's a paddling
throw new Error('New config failed to cache, apparently!')
}
@ -1293,7 +1304,7 @@ describe('service-worker', async () => {
// cached
assertEquals(
JSON.parse(
await window.waitForCacheAction(window.location.origin + 'test.json')),
await window.waitForCacheAction(window.location.origin + 'test.json', 'v1')),
{ test: "success" }
);
});
@ -1435,7 +1446,7 @@ describe('service-worker', async () => {
// let's see if it got added to cache
assertEquals(
JSON.parse(await window.waitForCacheAction(window.location.origin + 'test.json')),
JSON.parse(await window.waitForCacheAction(window.location.origin + 'test.json', 'v1')),
{ test: "success" }
);
});
@ -1551,7 +1562,7 @@ describe('service-worker', async () => {
// let's see if it got added to cache
assertEquals(
JSON.parse(await window.waitForCacheAction(window.location.origin + 'test.json')),
JSON.parse(await window.waitForCacheAction(window.location.origin + 'test.json', 'v1')),
{ test: "success" }
);
@ -1565,7 +1576,7 @@ describe('service-worker', async () => {
// let's see if it got removed from cache
assertEquals(
await window.waitForCacheAction(window.location.origin + 'test.json', "remove"),
await window.waitForCacheAction(window.location.origin + 'test.json', 'v1', "remove"),
undefined
);
});

Wyświetl plik

@ -237,13 +237,13 @@ let verifyConfigData = (cdata) => {
* configURL - url of the config file
* cresponse - response we're caching
*/
let cacheConfigJSON = async (configURL, cresponse) => {
let cacheConfigJSON = async (configURL, cresponse, use_cache) => {
try {
var cache = await caches.open('v1')
var cache = await caches.open(use_cache)
await cache.put(configURL, cresponse)
self.log('service-worker', 'config cached.')
self.log('service-worker', `config cached in cache: ${use_cache}.`)
} catch(e) {
self.log('service-worker', `failed to cache config: ${e}`)
self.log('service-worker', `failed to cache config in cache ${use_cache}: ${e}`)
}
}
@ -278,127 +278,143 @@ let getConfigJSON = async (cresponse) => {
* load plugin modules, making constructors available
* cycle through the plugin config instantiating plugins and their dependencies
*/
let executeConfig = (pluginsConfig) => {
// clean version of LibResilientPlugins
// NOTICE: this assumes LibResilientPlugins is not ever used *befure* this runs
// NOTICE: this assumption seems to hold currently, but noting for clarity
self.LibResilientPlugins = new Array()
let executeConfig = (config) => {
// this is the stash for plugins that need dependencies instantiated first
let dependentPlugins = new Array()
// working on a copy of the plugins config so that config.plugins remains unmodified
// in case we need it later (for example, when re-loading the config)
let pluginsConfig = [...config.plugins]
// only now load the plugins (config.json could have changed the defaults)
while (pluginsConfig.length > 0) {
// we want to catch any and all possible errors here
try {
// get the first plugin config from the array
let pluginConfig = pluginsConfig.shift()
self.log('service-worker', `handling plugin type: ${pluginConfig.name}`)
// this is the stash for plugins that need dependencies instantiated first
let dependentPlugins = new Array()
// load the relevant plugin script (if not yet loaded)
if (!LibResilientPluginConstructors.has(pluginConfig.name)) {
self.log('service-worker', `${pluginConfig.name}: loading plugin's source`)
self.importScripts(`./plugins/${pluginConfig.name}/index.js`)
}
// do we have any dependencies we should handle first?
if (typeof pluginConfig.uses !== "undefined") {
self.log('service-worker', `${pluginConfig.name}: ${pluginConfig.uses.length} dependencies found`)
// only now load the plugins (config.json could have changed the defaults)
while (pluginsConfig.length > 0) {
// move the dependency plugin configs to LibResilientConfig to be worked on next
for (let i=(pluginConfig.uses.length); i--; i>=0) {
self.log('service-worker', `${pluginConfig.name}: dependency found: ${pluginConfig.uses[i].name}`)
// put the plugin config in front of the plugin configs array
pluginsConfig.unshift(pluginConfig.uses[i])
// set each dependency plugin config to false so that we can keep track
// as we fill those gaps later with instantiated dependency plugins
pluginConfig.uses[i] = false
// get the first plugin config from the array
let pluginConfig = pluginsConfig.shift()
self.log('service-worker', `handling plugin type: ${pluginConfig.name}`)
// load the relevant plugin script (if not yet loaded)
if (!LibResilientPluginConstructors.has(pluginConfig.name)) {
self.log('service-worker', `${pluginConfig.name}: loading plugin's source`)
self.importScripts(`./plugins/${pluginConfig.name}/index.js`)
}
// stash the plugin config until we have all the dependencies handled
self.log('service-worker', `${pluginConfig.name}: not instantiating until dependencies are ready`)
dependentPlugins.push(pluginConfig)
// move on to the next plugin config, which at this point will be
// the first of dependencies for the plugin whose config got stashed
continue;
}
do {
// if the plugin is not enabled, no instantiation for it nor for its dependencies
// if the pluginConfig does not have an "enabled" field, it should be assumed to be "true"
if ( ( "enabled" in pluginConfig ) && ( pluginConfig.enabled != true ) ) {
self.log('service-worker', `skipping ${pluginConfig.name} instantiation: plugin not enabled (dependencies will also not be instantiated)`)
pluginConfig = dependentPlugins.pop()
if (pluginConfig !== undefined) {
let didx = pluginConfig.uses.indexOf(false)
pluginConfig.uses.splice(didx, 1)
// do we have any dependencies we should handle first?
if (typeof pluginConfig.uses !== "undefined") {
self.log('service-worker', `${pluginConfig.name}: ${pluginConfig.uses.length} dependencies found`)
// move the dependency plugin configs to LibResilientConfig to be worked on next
for (let i=(pluginConfig.uses.length); i--; i>=0) {
self.log('service-worker', `${pluginConfig.name}: dependency found: ${pluginConfig.uses[i].name}`)
// put the plugin config in front of the plugin configs array
pluginsConfig.unshift(pluginConfig.uses[i])
// set each dependency plugin config to false so that we can keep track
// as we fill those gaps later with instantiated dependency plugins
pluginConfig.uses[i] = false
}
// stash the plugin config until we have all the dependencies handled
self.log('service-worker', `${pluginConfig.name}: not instantiating until dependencies are ready`)
dependentPlugins.push(pluginConfig)
// move on to the next plugin config, which at this point will be
// the first of dependencies for the plugin whose config got stashed
continue;
}
// instantiate the plugin
let plugin = LibResilientPluginConstructors.get(pluginConfig.name)(self, pluginConfig)
self.log('service-worker', `${pluginConfig.name}: instantiated`)
do {
// if the plugin is not enabled, no instantiation for it nor for its dependencies
// if the pluginConfig does not have an "enabled" field, it should be assumed to be "true"
if ( ( "enabled" in pluginConfig ) && ( pluginConfig.enabled != true ) ) {
self.log('service-worker', `skipping ${pluginConfig.name} instantiation: plugin not enabled (dependencies will also not be instantiated)`)
pluginConfig = dependentPlugins.pop()
if (pluginConfig !== undefined) {
let didx = pluginConfig.uses.indexOf(false)
pluginConfig.uses.splice(didx, 1)
}
continue;
}
// instantiate the plugin
let plugin = LibResilientPluginConstructors.get(pluginConfig.name)(self, pluginConfig)
self.log('service-worker', `${pluginConfig.name}: instantiated`)
// do we have a stashed plugin that requires dependencies?
if (dependentPlugins.length === 0) {
// no we don't; so, this plugin goes directly to the plugin list
self.LibResilientPlugins.push(plugin)
// we're done here
self.log('service-worker', `${pluginConfig.name}: no dependent plugins, pushing directly to LibResilientPlugins`)
break;
}
// at this point clearly there is at least one element in dependentPlugins
// so we can safely assume that the freshly instantiated plugin is a dependency
//
// in that case let's find the first empty spot for a dependency
let didx = dependentPlugins[dependentPlugins.length - 1].uses.indexOf(false)
// assign the freshly instantiated plugin as that dependency
dependentPlugins[dependentPlugins.length - 1].uses[didx] = plugin
self.log('service-worker', `${pluginConfig.name}: assigning as dependency (#${didx}) to ${dependentPlugins[dependentPlugins.length - 1].name}`)
// was this the last one?
if (didx >= dependentPlugins[dependentPlugins.length - 1].uses.length - 1) {
// yup, last one!
self.log('service-worker', `${pluginConfig.name}: this was the last dependency of ${dependentPlugins[dependentPlugins.length - 1].name}`)
// we can now proceed to instantiate the last element of dependentPlugins
pluginConfig = dependentPlugins.pop()
continue
}
// it is not the last one, so there should be more dependency plugins to instantiate first
// before we can instantiate the last of element of dependentPlugins
// but that requires the full treatment, including checing the `uses` field for their configs
self.log('service-worker', `${pluginConfig.name}: not yet the last dependency of ${dependentPlugins[dependentPlugins.length - 1].name}`)
pluginConfig = false
// do we have a stashed plugin that requires dependencies?
if (dependentPlugins.length === 0) {
// no we don't; so, this plugin goes directly to the plugin list
self.LibResilientPlugins.push(plugin)
// we're done here
self.log('service-worker', `${pluginConfig.name}: no dependent plugins, pushing directly to LibResilientPlugins`)
break;
}
// if pluginConfig is not false, rinse-repeat the plugin instantiation steps
// since we are dealing with the last element of dependentPlugins
} while ( (pluginConfig !== false) && (pluginConfig !== undefined) )
// at this point clearly there is at least one element in dependentPlugins
// so we can safely assume that the freshly instantiated plugin is a dependency
//
// in that case let's find the first empty spot for a dependency
let didx = dependentPlugins[dependentPlugins.length - 1].uses.indexOf(false)
// assign the freshly instantiated plugin as that dependency
dependentPlugins[dependentPlugins.length - 1].uses[didx] = plugin
self.log('service-worker', `${pluginConfig.name}: assigning as dependency (#${didx}) to ${dependentPlugins[dependentPlugins.length - 1].name}`)
// was this the last one?
if (didx >= dependentPlugins[dependentPlugins.length - 1].uses.length - 1) {
// yup, last one!
self.log('service-worker', `${pluginConfig.name}: this was the last dependency of ${dependentPlugins[dependentPlugins.length - 1].name}`)
// we can now proceed to instantiate the last element of dependentPlugins
pluginConfig = dependentPlugins.pop()
continue
}
// it is not the last one, so there should be more dependency plugins to instantiate first
// before we can instantiate the last of element of dependentPlugins
// but that requires the full treatment, including checing the `uses` field for their configs
self.log('service-worker', `${pluginConfig.name}: not yet the last dependency of ${dependentPlugins[dependentPlugins.length - 1].name}`)
pluginConfig = false
// if pluginConfig is not false, rinse-repeat the plugin instantiation steps
// since we are dealing with the last element of dependentPlugins
} while ( (pluginConfig !== false) && (pluginConfig !== undefined) )
}
// finally -- do we want to use MIME type guessing based on content?
// dealing with this at the very end so that we know we can safely set detectMimeFromBuffer
// and not need to re-set it back in case anything fails
if (self.LibResilientConfig.useMimeSniffingLibrary === true) {
// we do not want to hit a NetworkError and end up using the default config
// much better to end up not using the fancy MIME type detection in such a case
try {
// we do! load the external lib
self.importScripts(`./lib/file-type.js`)
} catch (e) {
self.log('service-worker', `error when fetching external MIME sniffing library: ${e.message}`)
}
if (typeof fileType !== 'undefined' && "fileTypeFromBuffer" in fileType) {
detectMimeFromBuffer = fileType.fileTypeFromBuffer
self.log('service-worker', 'loaded external MIME sniffing library')
} else {
self.log('service-worker', 'failed to load external MIME sniffing library!')
// finally -- do we want to use MIME type guessing based on content?
// dealing with this at the very end so that we know we can safely set detectMimeFromBuffer
// and not need to re-set it back in case anything fails
if (config.useMimeSniffingLibrary === true) {
// we do not want to hit a NetworkError and end up using the default config
// much better to end up not using the fancy MIME type detection in such a case
try {
// we do! load the external lib
self.importScripts(`./lib/file-type.js`)
} catch (e) {
self.log('service-worker', `error when fetching external MIME sniffing library: ${e.message}`)
}
if (typeof fileType !== 'undefined' && "fileTypeFromBuffer" in fileType) {
detectMimeFromBuffer = fileType.fileTypeFromBuffer
self.log('service-worker', 'loaded external MIME sniffing library')
} else {
self.log('service-worker', 'failed to load external MIME sniffing library!')
}
}
// we're good!
return true;
// exception? no bueno
} catch (e) {
// inform
self.log('service-worker', `error while executing config: ${e.message}`)
// cleanup after a failed config execution
self.LibResilientPluginConstructors = new Map(lrpcBackup)
self.LibResilientPlugins = new Array()
// we are not good
return false;
}
}
@ -419,53 +435,18 @@ let initServiceWorker = async () => {
try {
// we'll need this later
var cresponse = null
let cresponse = null
let config = null
// get the config
//
// self.registration.scope contains the scope this service worker is registered for
// so it makes sense to pull config from `config.json` file directly under that location
try {
// config.json URL
var configURL = self.registration.scope + "config.json"
// get the config data from cache
cresponse = await caches.match(configURL)
var cdata = await getConfigJSON(cresponse)
// did it work?
if (cdata != false) {
// we need to know if the config was already cached
self.log('service-worker', `valid config file retrieved from cache.`)
// cache failed to deliver
} else {
// try fetching directly from the main domain
self.log('service-worker', `config file not found in cache, fetching.`)
var cresponse = await fetch(configURL)
cdata = await getConfigJSON(cresponse)
// did that work?
if (cdata != false) {
// cache it, asynchronously
cacheConfigJSON(configURL, cresponse)
} else {
// we ain't got nothing useful -- just set cdata to an empty object
cdata = {}
self.log('service-worker', 'ignoring invalid config, using defaults.')
}
}
// merge configs
self.LibResilientConfig = {...self.LibResilientConfig, ...cdata}
self.log('service-worker', 'config loaded.')
} catch (e) {
self.log('service-worker', 'config loading failed, using defaults; error:', e)
}
// TODO: this should probably be configurable somehow
let configURL = self.registration.scope + "config.json"
// clean version of LibResilientPlugins
// NOTICE: this assumes LibResilientPlugins is not ever used *before* this runs
// NOTICE: this assumption seems to hold currently, but noting for clarity
self.LibResilientPlugins = new Array()
// create the LibResilientPluginConstructors map
// the global... hack is here so that we can run tests; not the most elegant
@ -473,17 +454,106 @@ let initServiceWorker = async () => {
self.LibResilientPluginConstructors = self.LibResilientPluginConstructors || new Map()
// point backup of LibResilientPluginConstructors, in case we need to roll back
// this is used during cleanup in executeConfig()
// TODO: handle in a more elegant way
let lrpcBackup = new Map(self.LibResilientPluginConstructors)
try {
// working on a copy of the plugins config so that
// self.LibResilientConfig.plugins remains unmodified
// in case we need it later (for example, when re-loading the config)
executeConfig([...self.LibResilientConfig.plugins])
} catch (e) {
// cleanup after a failed config execution
self.LibResilientPluginConstructors = new Map(lrpcBackup)
// caches to try: temp cache, main cache
let available_caches = ['v1:temp', 'v1']
// keep track
let config_executed = false
let use_cache = false
do {
// init config data var
let cdata = false
// where are we getting the config.json from this time?
// we eitehr get a string (name of a cache), or undefined (signifying need for fetch())
use_cache = available_caches.shift()
try {
// cache?
if ( typeof use_cache === 'string' ) {
self.log('service-worker', `retrieving config.json from cache: ${use_cache}.`)
cresponse = await caches.match(configURL, {cacheName: use_cache})
// bail early if we got nothing
if (cresponse === undefined) {
self.log('service-worker', `config.json not found in cache: ${use_cache}.`)
continue
}
// regular fetch
// (we don't have any plugin transports at this point, obviously...)
} else {
self.log('service-worker', `retrieving config.json using fetch().`)
cresponse = await fetch(configURL)
}
// extract the JSON and verify it
cdata = await getConfigJSON(cresponse)
// exception? no bueno!
} catch(e) {
cdata = false
self.log('service-worker', `exception when trying to retrieve config.json: ${e.message}`)
}
// do we have anything to work with?
if (cdata === false) {
// cached config.json was invalid; no biggie, try another cache, or fetch()
if (typeof use_cache === "string") {
self.log('service-worker', `cached config.json is not valid; cache: ${use_cache}`)
continue
// if that was a fetch() config, we need to run to defaults!
} else {
self.log('service-worker', `fetched config.json is not valid; using defaults`)
// set an empty object, this will in effect deploy pure defaults
cdata = {}
// clear cresponse which will indicate later on
// that we did not use data from any response, cache nor fetch
cresponse = false
}
// we good!
} else {
self.log('service-worker', `valid-looking config.json retrieved.`)
}
// merge configs
config = {...self.LibResilientConfig, ...cdata}
// try executing the config
config_executed = executeConfig(config)
// if we're using the defaults, and yet loading of the config failed
// something is massively wrong
if ( ( cresponse === false ) && ( config_executed === false ) ) {
// this really should never happen
throw new Error('Failed to load the default config; this should never happen!')
}
// NOTICE: endless loop alert?
// NOTICE: this is not an endless loop because cresponse can only become false if we're using the default config
// NOTICE: and that single case is handled directly above
} while ( ! config_executed )
// we're good
self.LibResilientConfig = config
self.log('service-worker', 'config loaded.')
// we're good, let's cache the config if we need to
// that is, if it comes from the "v1:temp" cache
// or, was fetch()ed and valid (no caching if we're going with defaults, obviously)
if ( (use_cache === "v1:temp") || ( (use_cache === undefined) && (cresponse !== false) ) ) {
self.log('service-worker', `successfully loaded config.json; caching in cache: v1`)
cacheConfigJSON(configURL, cresponse, 'v1')
}
// inform
@ -491,7 +561,7 @@ let initServiceWorker = async () => {
initDone = true;
// regardless how we got the config file, if it's older than 24h...
if ( (new Date()) - Date.parse(cresponse.headers.get('date')) > 86400000) {
if ( ( cresponse !== false ) && (new Date()) - Date.parse(cresponse.headers.get('date')) > 86400000) {
// try to get it asynchronously through the plugins, and cache it
self.log('service-worker', `config.json stale, fetching through plugins`)
@ -503,7 +573,13 @@ let initServiceWorker = async () => {
var cdata = await getConfigJSON(cresponse)
// did that work?
if (cdata != false) {
if (cdata === false) {
// we got a false in cdata, that means it probably is invalid (or the fetch failed)
self.log('service-worker', 'config.json loaded through transport other than fetch seems invalid, ignoring')
return false
// otherwise, we good for more in-depth testing!
} else {
// if we got the new config.json via a method *other* than plain old fetch(),
// we will not be able to use importScripts() to load any pugins that have not been loaded already
@ -512,12 +588,6 @@ let initServiceWorker = async () => {
// which signifies that relevant code has been successfully loaded; but there are other failure modes!
if (cresponse.headers.get('x-libresilient-method') != 'fetch') {
// basic structure tests
if ( !verifyConfigData(cdata) ) {
self.log('service-worker', 'config.json loaded through transport other than fetch is invalid, ignoring')
return false
}
// go through the plugins in the new config and check if we already have their constructors
// i.e. if the plugin scripts have already been loaded
let currentPlugin = cdata.plugins.shift()
@ -549,9 +619,10 @@ let initServiceWorker = async () => {
} while ( (currentPlugin !== false) && (currentPlugin !== undefined) )
}
self.log('service-worker', `config.json successfully retrieved through plugins; caching.`)
// cache it, asynchronously
cacheConfigJSON(configURL, cresponse)
self.log('service-worker', `valid config.json successfully retrieved through plugins; caching.`)
// cache it, asynchronously, in the temporary cache
// as the config has not been "execute-tested" yet
cacheConfigJSON(configURL, cresponse, "v1:temp")
}
})
}