Merge branch 'wip-improved-config-handling' into 'master'

Milestone: Improved config.json handling when fetched via alternative channels

See merge request rysiekpl/libresilient!25
merge-requests/23/merge
Michał "rysiek" Woźniak 2024-02-11 23:39:51 +00:00
commit 124faa79e7
2 zmienionych plików z 1023 dodań i 416 usunięć

Wyświetl plik

@ -2,11 +2,6 @@
* LibResilient Service Worker.
*/
// initialize the LibResilientPlugins array
if (!Array.isArray(self.LibResilientPlugins)) {
self.LibResilientPlugins = new Array()
}
// initialize the LibResilientConfig array
//
// this also sets some sane defaults,
@ -200,6 +195,11 @@ self.guessMimeType = async function(ext, content) {
* cdata - config data to verify
*/
let verifyConfigData = (cdata) => {
// cdata needs to be an object
if ( typeof cdata !== "object" || cdata === null ) {
self.log('service-worker', 'fetched config does not contain a valid JSON object')
return false;
}
// basic check for the plugins field
if ( !("plugins" in cdata) || ! Array.isArray(cdata.plugins) ) {
self.log('service-worker', 'fetched config does not contain a valid "plugins" field')
@ -242,13 +242,13 @@ let verifyConfigData = (cdata) => {
* configURL - url of the config file
* cresponse - response we're caching
*/
let cacheConfigJSON = async (configURL, cresponse) => {
let cacheConfigJSON = async (configURL, cresponse, use_cache) => {
try {
var cache = await caches.open('v1')
var cache = await caches.open(use_cache)
await cache.put(configURL, cresponse)
self.log('service-worker', 'config cached.')
self.log('service-worker', `config cached in cache: ${use_cache}.`)
} catch(e) {
self.log('service-worker', `failed to cache config: ${e}`)
self.log('service-worker', `failed to cache config in cache ${use_cache}: ${e}`)
}
}
@ -277,7 +277,141 @@ let getConfigJSON = async (cresponse) => {
}
// flag signifying the SW has been initialized already
/**
* execute on the configuration
*
* load plugin modules, making constructors available
* cycle through the plugin config instantiating plugins and their dependencies
*/
let executeConfig = (config) => {
// working on a copy of the plugins config so that config.plugins remains unmodified
// in case we need it later (for example, when re-loading the config)
let pluginsConfig = [...config.plugins]
// this is the stash for plugins that need dependencies instantiated first
let dependentPlugins = new Array()
// only now load the plugins (config.json could have changed the defaults)
while (pluginsConfig.length > 0) {
// get the first plugin config from the array
let pluginConfig = pluginsConfig.shift()
self.log('service-worker', `handling plugin type: ${pluginConfig.name}`)
// load the relevant plugin script (if not yet loaded)
if (!LibResilientPluginConstructors.has(pluginConfig.name)) {
self.log('service-worker', `${pluginConfig.name}: loading plugin's source`)
self.importScripts(`./plugins/${pluginConfig.name}/index.js`)
}
// do we have any dependencies we should handle first?
if (typeof pluginConfig.uses !== "undefined") {
self.log('service-worker', `${pluginConfig.name}: ${pluginConfig.uses.length} dependencies found`)
// move the dependency plugin configs to LibResilientConfig to be worked on next
for (let i=(pluginConfig.uses.length); i--; i>=0) {
self.log('service-worker', `${pluginConfig.name}: dependency found: ${pluginConfig.uses[i].name}`)
// put the plugin config in front of the plugin configs array
pluginsConfig.unshift(pluginConfig.uses[i])
// set each dependency plugin config to false so that we can keep track
// as we fill those gaps later with instantiated dependency plugins
pluginConfig.uses[i] = false
}
// stash the plugin config until we have all the dependencies handled
self.log('service-worker', `${pluginConfig.name}: not instantiating until dependencies are ready`)
dependentPlugins.push(pluginConfig)
// move on to the next plugin config, which at this point will be
// the first of dependencies for the plugin whose config got stashed
continue;
}
do {
// if the plugin is not enabled, no instantiation for it nor for its dependencies
// if the pluginConfig does not have an "enabled" field, it should be assumed to be "true"
if ( ( "enabled" in pluginConfig ) && ( pluginConfig.enabled != true ) ) {
self.log('service-worker', `skipping ${pluginConfig.name} instantiation: plugin not enabled (dependencies will also not be instantiated)`)
pluginConfig = dependentPlugins.pop()
if (pluginConfig !== undefined) {
let didx = pluginConfig.uses.indexOf(false)
pluginConfig.uses.splice(didx, 1)
}
continue;
}
// instantiate the plugin
let plugin = LibResilientPluginConstructors.get(pluginConfig.name)(self, pluginConfig)
self.log('service-worker', `${pluginConfig.name}: instantiated`)
// do we have a stashed plugin that requires dependencies?
if (dependentPlugins.length === 0) {
// no we don't; so, this plugin goes directly to the plugin list
self.LibResilientPlugins.push(plugin)
// we're done here
self.log('service-worker', `${pluginConfig.name}: no dependent plugins, pushing directly to LibResilientPlugins`)
break;
}
// at this point clearly there is at least one element in dependentPlugins
// so we can safely assume that the freshly instantiated plugin is a dependency
//
// in that case let's find the first empty spot for a dependency
let didx = dependentPlugins[dependentPlugins.length - 1].uses.indexOf(false)
// assign the freshly instantiated plugin as that dependency
dependentPlugins[dependentPlugins.length - 1].uses[didx] = plugin
self.log('service-worker', `${pluginConfig.name}: assigning as dependency (#${didx}) to ${dependentPlugins[dependentPlugins.length - 1].name}`)
// was this the last one?
if (didx >= dependentPlugins[dependentPlugins.length - 1].uses.length - 1) {
// yup, last one!
self.log('service-worker', `${pluginConfig.name}: this was the last dependency of ${dependentPlugins[dependentPlugins.length - 1].name}`)
// we can now proceed to instantiate the last element of dependentPlugins
pluginConfig = dependentPlugins.pop()
continue
}
// it is not the last one, so there should be more dependency plugins to instantiate first
// before we can instantiate the last of element of dependentPlugins
// but that requires the full treatment, including checing the `uses` field for their configs
self.log('service-worker', `${pluginConfig.name}: not yet the last dependency of ${dependentPlugins[dependentPlugins.length - 1].name}`)
pluginConfig = false
// if pluginConfig is not false, rinse-repeat the plugin instantiation steps
// since we are dealing with the last element of dependentPlugins
} while ( (pluginConfig !== false) && (pluginConfig !== undefined) )
}
// finally -- do we want to use MIME type guessing based on content?
// dealing with this at the very end so that we know we can safely set detectMimeFromBuffer
// and not need to re-set it back in case anything fails
if (config.useMimeSniffingLibrary === true) {
// we do not want to hit a NetworkError and end up using the default config
// much better to end up not using the fancy MIME type detection in such a case
try {
// we do! load the external lib
self.importScripts(`./lib/file-type.js`)
} catch (e) {
self.log('service-worker', `error when fetching external MIME sniffing library: ${e.message}`)
}
if (typeof fileType !== 'undefined' && "fileTypeFromBuffer" in fileType) {
detectMimeFromBuffer = fileType.fileTypeFromBuffer
self.log('service-worker', 'loaded external MIME sniffing library')
} else {
self.log('service-worker', 'failed to load external MIME sniffing library!')
}
}
// we're good!
return true;
}
// flag signifying if the SW has been initialized already
var initDone = false
// load the plugins
@ -292,170 +426,152 @@ let initServiceWorker = async () => {
try {
// we'll need this later
var cresponse = null
let cresponse = null
let config = null
// get the config
//
// self.registration.scope contains the scope this service worker is registered for
// so it makes sense to pull config from `config.json` file directly under that location
try {
// config.json URL
var configURL = self.registration.scope + "config.json"
// get the config data from cache
cresponse = await caches.match(configURL)
var cdata = await getConfigJSON(cresponse)
// did it work?
if (cdata != false) {
// we need to know if the config was already cached
self.log('service-worker', `valid config file retrieved from cache.`)
// cache failed to deliver
} else {
// try fetching directly from the main domain
self.log('service-worker', `config file not found in cache, fetching.`)
var cresponse = await fetch(configURL)
cdata = await getConfigJSON(cresponse)
// did that work?
if (cdata != false) {
// cache it, asynchronously
cacheConfigJSON(configURL, cresponse)
} else {
// we ain't got nothing useful -- just set cdata to an empty object
cdata = {}
self.log('service-worker', 'ignoring invalid config, using defaults.')
}
}
// merge configs
self.LibResilientConfig = {...self.LibResilientConfig, ...cdata}
self.log('service-worker', 'config loaded.')
} catch (e) {
self.log('service-worker', 'config loading failed, using defaults; error:', e)
}
// TODO: this should probably be configurable somehow
let configURL = self.registration.scope + "config.json"
// first let's deal with the easy part -- do we want to use MIME type guessing based on content?
if (self.LibResilientConfig.useMimeSniffingLibrary === true) {
// we do! load the external lib
self.importScripts(`./lib/file-type.js`)
if (typeof fileType !== 'undefined' && "fileTypeFromBuffer" in fileType) {
detectMimeFromBuffer = fileType.fileTypeFromBuffer
self.log('service-worker', 'loaded external MIME sniffing library')
} else {
self.log('service-worker', 'failed to load external MIME sniffing library!')
}
}
// clean version of LibResilientPlugins
// NOTICE: this assumes LibResilientPlugins is not ever used *before* this runs
// NOTICE: this assumption seems to hold currently, but noting for clarity
self.LibResilientPlugins = new Array()
// create the LibResilientPluginConstructors map
// the global... hack is here so that we can run tests; not the most elegant
// TODO: find a better way
self.LibResilientPluginConstructors = self.LibResilientPluginConstructors || new Map()
// copy of the plugins config
// we need to work on it so that self.LibResilientConfig.plugins remains unmodified
// in case we need it later (for example, when re-loading the config)
var pluginsConfig = [...self.LibResilientConfig.plugins]
// point backup of LibResilientPluginConstructors, in case we need to roll back
// this is used during cleanup in executeConfig()
// TODO: handle in a more elegant way
let lrpcBackup = new Map(self.LibResilientPluginConstructors)
// this is the stash for plugins that need dependencies instantiated first
var dependentPlugins = new Array()
// caches to try: temp cache, main cache
let available_caches = ['v1', 'v1:verified']
// only now load the plugins (config.json could have changed the defaults)
while (pluginsConfig.length > 0) {
// keep track
let config_executed = false
let use_cache = false
do {
// get the first plugin config from the array
let pluginConfig = pluginsConfig.shift()
self.log('service-worker', `handling plugin type: ${pluginConfig.name}`)
// init config data var
let cdata = false
// load the relevant plugin script (if not yet loaded)
if (!LibResilientPluginConstructors.has(pluginConfig.name)) {
self.log('service-worker', `${pluginConfig.name}: loading plugin's source`)
self.importScripts(`./plugins/${pluginConfig.name}/index.js`)
}
// where are we getting the config.json from this time?
// we eitehr get a string (name of a cache), or undefined (signifying need for fetch())
use_cache = available_caches.shift()
// do we have any dependencies we should handle first?
if (typeof pluginConfig.uses !== "undefined") {
self.log('service-worker', `${pluginConfig.name}: ${pluginConfig.uses.length} dependencies found`)
try {
// move the dependency plugin configs to LibResilientConfig to be worked on next
for (var i=(pluginConfig.uses.length); i--; i>=0) {
self.log('service-worker', `${pluginConfig.name}: dependency found: ${pluginConfig.uses[i].name}`)
// put the plugin config in front of the plugin configs array
pluginsConfig.unshift(pluginConfig.uses[i])
// set each dependency plugin config to false so that we can keep track
// as we fill those gaps later with instantiated dependency plugins
pluginConfig.uses[i] = false
}
// stash the plugin config until we have all the dependencies handled
self.log('service-worker', `${pluginConfig.name}: not instantiating until dependencies are ready`)
dependentPlugins.push(pluginConfig)
// move on to the next plugin config, which at this point will be
// the first of dependencies for the plugin whose config got stashed
continue;
}
do {
// if the plugin is not enabled, no instantiation for it nor for its dependencies
// if the pluginConfig does not have an "enabled" field, it should be assumed to be "true"
if ( ( "enabled" in pluginConfig ) && ( pluginConfig.enabled != true ) ) {
self.log('service-worker', `skipping ${pluginConfig.name} instantiation: plugin not enabled (dependencies will also not be instantiated)`)
pluginConfig = dependentPlugins.pop()
if (pluginConfig !== undefined) {
let didx = pluginConfig.uses.indexOf(false)
pluginConfig.uses.splice(didx, 1)
// cache?
if ( typeof use_cache === 'string' ) {
self.log('service-worker', `retrieving config.json from cache: ${use_cache}.`)
cresponse = await caches.match(configURL, {cacheName: use_cache})
// bail early if we got nothing
if (cresponse === undefined) {
self.log('service-worker', `config.json not found in cache: ${use_cache}.`)
continue
}
continue;
// regular fetch
// (we don't have any plugin transports at this point, obviously...)
} else {
self.log('service-worker', `retrieving config.json using fetch().`)
cresponse = await fetch(configURL)
}
// instantiate the plugin
let plugin = LibResilientPluginConstructors.get(pluginConfig.name)(self, pluginConfig)
self.log('service-worker', `${pluginConfig.name}: instantiated`)
// extract the JSON and verify it
cdata = await getConfigJSON(cresponse)
// do we have a stashed plugin that requires dependencies?
if (dependentPlugins.length === 0) {
// no we don't; so, this plugin goes directly to the plugin list
self.LibResilientPlugins.push(plugin)
// we're done here
self.log('service-worker', `${pluginConfig.name}: no dependent plugins, pushing directly to LibResilientPlugins`)
break;
}
// exception? no bueno!
} catch(e) {
cdata = false
self.log('service-worker', `exception when trying to retrieve config.json: ${e.message}`)
}
// at this point clearly there is at least one element in dependentPlugins
// so we can safely assume that the freshly instantiated plugin is a dependency
//
// in that case let's find the first empty spot for a dependency
let didx = dependentPlugins[dependentPlugins.length - 1].uses.indexOf(false)
// assign the freshly instantiated plugin as that dependency
dependentPlugins[dependentPlugins.length - 1].uses[didx] = plugin
self.log('service-worker', `${pluginConfig.name}: assigning as dependency (#${didx}) to ${dependentPlugins[dependentPlugins.length - 1].name}`)
// do we have anything to work with?
if (cdata === false) {
// was this the last one?
if (didx >= dependentPlugins[dependentPlugins.length - 1].uses.length - 1) {
// yup, last one!
self.log('service-worker', `${pluginConfig.name}: this was the last dependency of ${dependentPlugins[dependentPlugins.length - 1].name}`)
// we can now proceed to instantiate the last element of dependentPlugins
pluginConfig = dependentPlugins.pop()
// cached config.json was invalid; no biggie, try another cache, or fetch()
if (typeof use_cache === "string") {
self.log('service-worker', `cached config.json is not valid; cache: ${use_cache}`)
continue
// if that was a fetch() config, we need to run to defaults!
} else {
self.log('service-worker', `fetched config.json is not valid; using defaults`)
// set an empty object, this will in effect deploy pure defaults
cdata = {}
// clear cresponse which will indicate later on
// that we did not use data from any response, cache nor fetch
cresponse = false
}
// it is not the last one, so there should be more dependency plugins to instantiate first
// before we can instantiate the last of element of dependentPlugins
// but that requires the full treatment, including checing the `uses` field for their configs
self.log('service-worker', `${pluginConfig.name}: not yet the last dependency of ${dependentPlugins[dependentPlugins.length - 1].name}`)
pluginConfig = false
// we good!
} else {
self.log('service-worker', `valid-looking config.json retrieved.`)
}
// merge configs
config = {...self.LibResilientConfig, ...cdata}
// try executing the config
// we want to catch any and all possible errors here
try {
config_executed = executeConfig(config)
// exception? no bueno
} catch (e) {
// inform
self.log('service-worker', `error while executing config: ${e.message}`)
// cleanup after a failed config execution
self.LibResilientPluginConstructors = new Map(lrpcBackup)
self.LibResilientPlugins = new Array()
// we are not good
config_executed = false;
}
// if pluginConfig is not false, rinse-repeat the plugin instantiation steps
// since we are dealing with the last element of dependentPlugins
} while ( (pluginConfig !== false) && (pluginConfig !== undefined) )
// if we're using the defaults, and yet loading of the config failed
// something is massively wrong
if ( ( cresponse === false ) && ( config_executed === false ) ) {
// this really should never happen
throw new Error('Failed to load the default config; this should never happen!')
}
// NOTICE: endless loop alert?
// NOTICE: this is not an endless loop because cresponse can only become false if we're using the default config
// NOTICE: and that single case is handled directly above
} while ( ! config_executed )
// we're good
self.LibResilientConfig = config
self.log('service-worker', 'config loaded.')
// we're good, let's cache the config as verified if we need to
// that is, if it comes from the "v1" cache...
if (use_cache === "v1") {
self.log('service-worker', `successfully loaded config.json; caching in cache: v1:verified`)
await cacheConfigJSON(configURL, cresponse, 'v1:verified')
// we used the v1:verified cache; we should cache config.json into the v1 cache
// as that will speed things up a bit next time we need to load the service worker
} else if (use_cache === "v1:verified") {
self.log('service-worker', `successfully loaded config.json; caching in cache: v1`)
await cacheConfigJSON(configURL, cresponse, 'v1')
// or, was fetch()-ed and valid (no caching if we're going with defaults, obviously)
} else if ( (use_cache === undefined) && (cresponse !== false) ) {
self.log('service-worker', `successfully loaded config.json; caching in cache: v1, v1:verified`)
// we want to cache to both, so that:
// 1. we get the extra bit of performance from using the v1 cache that is checked first
// 2. but we get the verified config already in the v1:verified cache for later
await cacheConfigJSON(configURL, await cresponse.clone(), 'v1')
await cacheConfigJSON(configURL, cresponse, 'v1:verified')
}
// inform
@ -463,7 +579,7 @@ let initServiceWorker = async () => {
initDone = true;
// regardless how we got the config file, if it's older than 24h...
if ( (new Date()) - Date.parse(cresponse.headers.get('date')) > 86400000) {
if ( ( cresponse !== false ) && (new Date()) - Date.parse(cresponse.headers.get('date')) > 86400000) {
// try to get it asynchronously through the plugins, and cache it
self.log('service-worker', `config.json stale, fetching through plugins`)
@ -475,22 +591,28 @@ let initServiceWorker = async () => {
var cdata = await getConfigJSON(cresponse)
// did that work?
if (cdata != false) {
if (cdata === false) {
// we got a false in cdata, that means it probably is invalid (or the fetch failed)
self.log('service-worker', 'config.json loaded through transport other than fetch seems invalid, ignoring')
return false
// otherwise, we good for more in-depth testing!
} else {
// if we got the new config.json via a method *other* than plain old fetch(),
// we will not be able to use importScripts() to load any pugins that have not been loaded already
//
// NOTICE: this *only* checks if we have all the necessary plugin constructors already available
// which signifies that relevant code has been successfully loaded; but there are other failure modes!
if (cresponse.headers.get('x-libresilient-method') != 'fetch') {
// go through the plugins in the new config and check if we already have their constructors
// i.e. if the plugin scripts have already been loaded
// FIXME: this does not currently dive into dependencies!
// FIXME: https://gitlab.com/rysiekpl/libresilient/-/issues/48
for (let p in cdata.plugins) {
var pname = cdata.plugins[p].name
let currentPlugin = cdata.plugins.shift()
do {
// plugin constructor not available, meaning: we'd have to importScripts() it
// but we can't since this was not retrieved via fetch(), so we cannot assume
// that the main domain of the website is up and available
// that the main domain of the website is up and available
//
// if we cache this newly retrieved config.json, next time the service worker gets restarted
// we will end up with an error while trying to run importScripts() for this plugin
@ -499,16 +621,26 @@ let initServiceWorker = async () => {
// if the main domain is not available, this would mean the website stops working
// even though we *were* able to retrieve the new config.json via plugins!
// so, ignoring this new config.json.
if (!LibResilientPluginConstructors.has(pname)) {
self.log('service-worker', `config.json was retrieved through plugins other than fetch, but specifies additional plugins (${pname}); ignoring.`)
if (!LibResilientPluginConstructors.has(currentPlugin.name)) {
self.log(
'service-worker',
`warning: config.json loaded through transport other than fetch, but specifies not previously loaded plugin: "${currentPlugin.name}"\nignoring the whole config.json.`)
return false;
}
}
// push any dependencies into the array, at the very front
// thus gradually flattening the config
if ("uses" in currentPlugin) {
cdata.plugins.unshift(...currentPlugin.uses)
}
// get the next plugin to check
currentPlugin = cdata.plugins.shift()
} while ( (currentPlugin !== false) && (currentPlugin !== undefined) )
}
self.log('service-worker', `config.json successfully retrieved through plugins; caching.`)
// cache it, asynchronously
cacheConfigJSON(configURL, cresponse)
self.log('service-worker', `valid config.json successfully retrieved through plugins; caching.`)
// cache it, asynchronously, in the temporary cache
// as the config has not been "execute-tested" yet
cacheConfigJSON(configURL, cresponse, "v1")
}
})
}
@ -517,10 +649,10 @@ let initServiceWorker = async () => {
// we only get a cryptic "Error while registering a service worker"
// unless we explicitly print the errors out in the console
console.error(e)
// we got an error while initializing the plugins
// we got an error while initializing the service worker!
// better play it safe!
self.registration.unregister()
throw e
return false
}
return true;
}
@ -1010,11 +1142,12 @@ let getResourceThroughLibResilient = (url, init, clientId, useStashed=true, doSt
|* === Setting up the event handlers === *|
\* ========================================================================= */
self.addEventListener('install', async (event) => {
await event.waitUntil(
initServiceWorker()
)
// "COMMIT_UNKNOWN" will be replaced with commit ID
self.log('service-worker', "0. Installed LibResilient Service Worker (commit: COMMIT_UNKNOWN).");
let init_promise = initServiceWorker()
await event.waitUntil(init_promise)
if (await init_promise === true) {
// "COMMIT_UNKNOWN" will be replaced with commit ID
self.log('service-worker', "0. Installed LibResilient Service Worker (commit: COMMIT_UNKNOWN).");
}
});
self.addEventListener('activate', async event => {