Merge branch 'wip-improve-for-admins' into 'master'

Improving experience for site administrators

See merge request rysiekpl/libresilient!21

__tests__/service-worker/service-worker.test.js

@@ -349,6 +349,7 @@ describe('service-worker', async () => {
         assertEquals(self.LibResilientConfig.defaultPluginTimeout, 10000)
         assertEquals(self.LibResilientConfig.plugins, [{name: "fetch"},{name: "cache"}])
         assertEquals(self.LibResilientConfig.loggedComponents, ['service-worker', 'fetch', 'cache'])
+        assertEquals(self.LibResilientConfig.normalizeQueryParams, true)
         assertSpyCalls(self.fetch, 1)
     })
     
@@ -367,6 +368,7 @@ describe('service-worker', async () => {
         assertEquals(self.LibResilientConfig.defaultPluginTimeout, 10000)
         assertEquals(self.LibResilientConfig.plugins, [{name: "fetch"},{name: "cache"}])
         assertEquals(self.LibResilientConfig.loggedComponents, ['service-worker', 'fetch', 'cache'])
+        assertEquals(self.LibResilientConfig.normalizeQueryParams, true)
         assertSpyCalls(self.fetch, 1)
     })
     
@@ -385,6 +387,7 @@ describe('service-worker', async () => {
         assertEquals(self.LibResilientConfig.defaultPluginTimeout, 10000)
         assertEquals(self.LibResilientConfig.plugins, [{name: "fetch"},{name: "cache"}])
         assertEquals(self.LibResilientConfig.loggedComponents, ['service-worker', 'fetch', 'cache'])
+        assertEquals(self.LibResilientConfig.normalizeQueryParams, true)
         assertSpyCalls(self.fetch, 1)
     })
     
@@ -403,6 +406,7 @@ describe('service-worker', async () => {
         assertEquals(self.LibResilientConfig.defaultPluginTimeout, 10000)
         assertEquals(self.LibResilientConfig.plugins, [{name: "fetch"},{name: "cache"}])
         assertEquals(self.LibResilientConfig.loggedComponents, ['service-worker', 'fetch', 'cache'])
+        assertEquals(self.LibResilientConfig.normalizeQueryParams, true)
         assertSpyCalls(self.fetch, 1)
     })
   
@@ -421,12 +425,32 @@ describe('service-worker', async () => {
         assertEquals(self.LibResilientConfig.defaultPluginTimeout, 10000)
         assertEquals(self.LibResilientConfig.plugins, [{name: "fetch"},{name: "cache"}])
         assertEquals(self.LibResilientConfig.loggedComponents, ['service-worker', 'fetch', 'cache'])
+        assertEquals(self.LibResilientConfig.normalizeQueryParams, true)
+        assertSpyCalls(self.fetch, 1)
+    })
+    
+    it("should use default LibResilientConfig values when 'normalizeQueryParams' field in config.json contains an invalid value", async () => {
+        
+        let mock_response_data = {
+            data: JSON.stringify({loggedComponents: ['service-worker', 'fetch'], plugins: [{name: "fetch"}], defaultPluginTimeout: 5000, normalizeQueryParams: "not a boolean"})
+        }
+        window.fetch = spy(window.getMockedFetch(mock_response_data))
+        
+        await import("../../service-worker.js?" + window.test_id);
+        await self.dispatchEvent(new Event('install'))
+        await self.waitForSWInstall()
+        
+        assertEquals(typeof self.LibResilientConfig, "object")
+        assertEquals(self.LibResilientConfig.defaultPluginTimeout, 10000)
+        assertEquals(self.LibResilientConfig.plugins, [{name: "fetch"},{name: "cache"}])
+        assertEquals(self.LibResilientConfig.loggedComponents, ['service-worker', 'fetch', 'cache'])
+        assertEquals(self.LibResilientConfig.normalizeQueryParams, true)
         assertSpyCalls(self.fetch, 1)
     })
     
     it("should use config values from a valid fetched config.json file, caching it", async () => {
         let mock_response_data = {
-            data: JSON.stringify({loggedComponents: ['service-worker', 'cache'], plugins: [{name: "cache"}], defaultPluginTimeout: 5000})
+            data: JSON.stringify({loggedComponents: ['service-worker', 'cache'], plugins: [{name: "cache"}], defaultPluginTimeout: 5000, normalizeQueryParams: false})
         }
         window.fetch = spy(window.getMockedFetch(mock_response_data))
         
@@ -438,6 +462,7 @@ describe('service-worker', async () => {
         assertEquals(self.LibResilientConfig.defaultPluginTimeout, 5000)
         assertEquals(self.LibResilientConfig.plugins, [{name: "cache"}])
         assertEquals(self.LibResilientConfig.loggedComponents, ['service-worker', 'cache'])
+        assertEquals(self.LibResilientConfig.normalizeQueryParams, false)
         assertSpyCalls(self.fetch, 1)
         
         // cacheConfigJSON() is called asynchronously in the Service Worker,
@@ -987,6 +1012,49 @@ describe('service-worker', async () => {
         assertEquals(await response.json(), { test: "success" })
     });
     
+    it("should normalize query params in requested URLs by default", async () => {
+        
+        console.log(self.LibResilientConfig)
+        
+        await import("../../service-worker.js?" + window.test_id);
+        await self.dispatchEvent(new Event('install'))
+        await self.waitForSWInstall()
+        
+        let fetch_event = new FetchEvent('test.json?b=bbb&a=aaa&d=ddd&c=ccc')
+        window.dispatchEvent(fetch_event)
+        let response = await fetch_event.waitForResponse()
+        
+        assertEquals(
+            fetch.calls[1].args[0],
+            "https://test.resilient.is/test.json?a=aaa&b=bbb&c=ccc&d=ddd"
+        )
+    })
+    
+    it("should not normalize query params in requested URLs if 'normalizeQueryParams' is set to false", async () => {
+        self.LibResilientConfig = {
+            plugins: [{
+                name: 'fetch'
+            }],
+            loggedComponents: [
+                'service-worker'
+            ],
+            normalizeQueryParams: false
+        }
+        
+        await import("../../service-worker.js?" + window.test_id);
+        await self.dispatchEvent(new Event('install'))
+        await self.waitForSWInstall()
+        
+        let fetch_event = new FetchEvent('test.json?b=bbb&a=aaa&d=ddd&c=ccc')
+        window.dispatchEvent(fetch_event)
+        let response = await fetch_event.waitForResponse()
+        
+        assertEquals(
+            fetch.calls[1].args[0],
+            "https://test.resilient.is/test.json?b=bbb&a=aaa&d=ddd&c=ccc"
+        )
+    })
+    
     it("should pass the Request() init data to plugins", async () => {
         self.LibResilientConfig = {
             plugins: [{

docs/FAQ.md

@@ -65,10 +65,12 @@ More information on the Service Worker API [is available in MDN](https://develop
 
 ### How do I update LibResilient's service worker code?
 
-For most use-cases it's enough to deploy the new code and let visitor's browser update the service worker automatically. Browsers of returning visitors will load the new version on the first visit that happens after the cache expires, no longer than 24 hours though (as per Service Workers API). LibResilient *does not* handle requests for the `service-worker.js` script nor any of the loaded plugins: in accordance to the API specification, a service worker cannot handle requests for its own code (this is enforced by the browser itself).
+For most use-cases it's enough to deploy the new code and let visitor's browser update the service worker automatically. Browsers of returning visitors will load the new version on the first visit that happens after the cache expires, no longer than 24 hours though (as per Service Workers API). LibResilient *does not* handle requests for the `service-worker.js` script nor for any of the loaded plugins: in accordance to the API specification, a service worker cannot handle requests for its own code (this is enforced by the browser itself).
 
 The update can also be triggered by the [`update()` method of the `ServiceWorkerRegistration`](https://developer.mozilla.org/en-US/docs/Web/API/ServiceWorkerRegistration/update) interface, but that too complies with the HTTP caching headers for caching up to 24 hours.
 
+It is not adviseable to change the name of the service worker script after deployment. This can lead to multiple service workers loaded in visitor's clients simultaneously.
+
 When debugging code on a developer machine, you can disable cache in your browser's developer tools, which will make the service worker script load as soon as it changes. Please remember that loading and activating a service worker are two different things! See the answer on debugging service workers lower in this FAQ.
 
 ### Can LibResilient's service worker be updated when the original domain is not accessible?
@@ -77,7 +79,9 @@ No, it cannot. This is clearly specified in the API. Until the website on the or
 
 In the context of LibResilient, this means that even when LibResilient is deployed on a website, and works as expected (pulling content from alternative endpoints or through alternative transports), as long as the original domain is down it is *impossible* to update the service worker or load new plugins.
 
-It is, however, possible to change LibResilient's configuration (including configuration of already loaded plugins), as `config.json` is not a JavaScript file and therefore it is *not* treated as code by browsers. You can read more on updating LibResilient's configuration during disruption [here](./UPDATING_DURING_DISRUPTION.md).
+Please note: if the original domain is hijacked or otehrwise taken over, whoever controls it can deploy their own service worker code that will be loaded by visitor's clients, as long as they use the same path and file name of the service worker script. In a situation where the original domain has been taken over or can be expected to be taken over, it is strongly advised to deploy configuration changes that redirect visitors to a new domain (for example, by using the [`redirect` plugin](../plugins/redirect/)).
+
+It is possible, with some caveats (see below), to change LibResilient's configuration (including configuration of already loaded plugins) even if the original domain is down, as `config.json` is not a JavaScript file and therefore it is *not* treated as code by browsers. You can read more on updating LibResilient's configuration during disruption [here](./UPDATING_DURING_DISRUPTION.md).
 
 ### Can LibResilient's plugins be updated, or new plugins loaded, when the original domain is not accessible?
 
@@ -97,11 +101,13 @@ A browser will periodically check if a new version of the registered service wor
 
 So a buggy service worker would be deployed for a given user at least for 24 hours (or longer, if they visit the website less frequently).
 
+You can use the [`remove-service-worker.js` script](../lib/remove-service-worker.js) to remove buggy or misconfigured LibResilient service workers from visitor's clients. To do so, replace the contents of `libresilient.js` and `service-worker.js`, as deployed on your site, with the contents of that script. Visitor's clients will periodically check if a new version of the service worker script is available, and eventually load the self-destructing service worker script. How long it takes depends on how often a given visitor visits your site, though.
+
 However, this only relates to the *code* of the service worker (and in case of LibResilient, the code of the plugins). As discussed above, LibResilient allows for *configuration* updates even if the main website is unreachable for whatever reason. Configuration changes (even during disruption) can *disable* plugins. In other words, in certain situations it might be possible to deploy a configuration-based fix to a problem (either by disabling a problematic plugin, or changing the problematic settings).
 
 ### If the original domain is down and the service worker is buggy, how long until it gets automatically removed by the browser?
 
-Eventually the browser will remove a service worker. It is unlcear after how long, exactly, but probably a few weeks. But if the site is not available anyway (for reasons unrelated to the service worker), this does not seem like a serious downside.
+Eventually the browser will remove a service worker. It is unlcear after how long, exactly, but probably a few weeks. If the site is not available anyway (for reasons unrelated to the service worker), this does not seem like a serious downside.
 
 If the site is available but the service worker code is buggy for whatever reason, it is possible to remove/update it (see above).
 

lib/remove-service-worker.js

@@ -0,0 +1,76 @@
+/*
+ * emergency removal of a mis-configured service worker
+ * 
+ * this file will un-register the service worker, attempting this both from client-side
+ * and by way of installing a self-destructing service worker script.
+ * 
+ * use this file to replace these two LibResilient files as deployed on your site,
+ * keeping their names, then publish your site as normal:
+ * 
+ * - service-worker.js
+ * - libresilient.js
+ * 
+ * these files are normally distributed in the directory that is parent to the one
+ * this file is in.
+ */
+
+// are we running client-side?
+if ('serviceWorker' in navigator) {
+    // we are. unregister all relevant service workers 
+    console.log('LibResilient [COMMIT_UNKNOWN, client-side] :: emergency service worker removal code started')
+    navigator.serviceWorker.getRegistrations().then( async (registrations)=>{
+        console.log(`LibResilient [COMMIT_UNKNOWN, client-side] :: removing ${registrations.length} registrations...`)
+        for (let registration of registrations) {
+            let result = await registration.unregister();
+            if (result === true) {
+                console.log('LibResilient [COMMIT_UNKNOWN, client-side] :: +-- unregistered a registration')
+            } else {
+                console.log('LibResilient [COMMIT_UNKNOWN, client-side] :: +-- failed to unregister a registration')
+            }
+        }
+    });
+    
+// otherwise, are we running as a service worker already?
+} else if ('registration' in self) {
+
+    self.addEventListener('install', () => {
+        console.log('LibResilient [COMMIT_UNKNOWN, service-worker] :: emergency service worker removal code installed')
+        // we want this activated immediately;
+        // this means that we become the service worker immediately
+        self.skipWaiting();
+    });
+
+    self.addEventListener('activate', () => {
+        // since we are the service worker, we can just... unregister ourselves
+        console.log('LibResilient [COMMIT_UNKNOWN, service-worker] :: unregistering the service worker...')
+        self.registration
+                    .unregister()
+                    .then((unresult)=>{
+                        // was the unregistration a success?
+                        if (unresult) {
+                            // indeed
+                            console.log('LibResilient [COMMIT_UNKNOWN, service-worker] :: successfully unregistered the service worker')
+                            // return the list of clients to reload in order to remove the last remnants of the service worker
+                            return self.clients.matchAll({
+                                type: 'window'
+                            })
+                        } else {
+                            // failed unregistration
+                            console.log('LibResilient [COMMIT_UNKNOWN, service-worker] :: failed to unregister the service worker')
+                            // return an empty array, as it's unclear if the service worker was unregistered
+                            return []
+                        }
+                    })
+                    // reload all clients (if any) 
+                    .then(wclients => {
+                        console.log(`LibResilient [COMMIT_UNKNOWN, service-worker] :: reloading ${wclients.length} clients`)
+                        wclients.forEach((wclient) => {
+                            console.log('LibResilient [COMMIT_UNKNOWN, service-worker] :: +-- client:', wclient)
+                            wclient.navigate(wclient.url);
+                        });
+                    })
+    });
+    
+} else {
+    console.warn("unable to load LibResilient: ServiceWorker API not available in the browser")
+}

plugins/basic-integrity/README.md

@@ -17,8 +17,10 @@ The `basic-integrity` plugin supports the following configuration options:
    Array containing exactly one object which is in turn a configuration of a wrapped plugin. This plugin will be used to actually handle any requests.
    
  - `integrity` (default: empty)  
-   An object mapping absolute URLs (e.g. "`https://example.com/img/test.png`") to integrity hashes (e.g. "`sha384-kn5dhxz4RpBmx7xC7Dmq2N43PclV9U/niyh+4Km7oz5W0FaWdz3Op+3K0Qxz8y3z`"). Supported integrity hash algorithms [as per SRI specification](https://w3c.github.io/webappsec-subresource-integrity/#terms): `sha256`, `sha384`, `sha512`.  
-   The integrity string can contain multiple hashes, space-separated, [as per the standard](https://w3c.github.io/webappsec-subresource-integrity/#agility).
+   An object mapping URLs (e.g. "`https://example.com/img/test.png`") and absolute paths (e.g. "`/img/test.png`") to integrity hashes (e.g. "`sha384-kn5dhxz4RpBmx7xC7Dmq2N43PclV9U/niyh+4Km7oz5W0FaWdz3Op+3K0Qxz8y3z`"). Supported integrity hash algorithms [as per SRI specification](https://w3c.github.io/webappsec-subresource-integrity/#terms): `sha256`, `sha384`, `sha512`.  
+   The integrity string can contain multiple hashes, space-separated, [as per the standard](https://w3c.github.io/webappsec-subresource-integrity/#agility).  
+   When integrity data is specified twice for the same effective URL (once using full URL, once using absolute path), it is concatenated before passing the request on to the wrapped plugin.  
+   Using absolute paths instead of URLs as keys is useful when hosting the same website content under multiple domain names, assuming the same paths are used across all domains.
    
  - `requireIntegrity` (default: `true`)  
    Boolean value specifying if integrity data is required for a request to handled. That is: if a request is being handled for a URL that does not have integrity data associated with it, should the request be processed, or errored out?  

plugins/basic-integrity/__tests__/browser.test.js

@@ -49,6 +49,11 @@ describe('browser: basic-integrity plugin', async () => {
                 console.debug(component + ' :: ', ...items)
             }
         }
+    // mocking window.location
+    // https://developer.mozilla.org/en-US/docs/Web/API/Window/location
+    window.location = {
+        origin: "https://resilient.is"
+    }
     window.resolvingFetch = (url, init) => {
                                 return Promise.resolve(
                                     new Response(
@@ -234,5 +239,54 @@ describe('browser: basic-integrity plugin', async () => {
             })
         assertEquals(await response.json(), {test: "success"})
     });
+    
+    it("should set integrity data specified for absolute paths correctly on relevant requests", async () => {
+        init.integrity = {
+            "/test.json": "sha384-kn5dhxz4RpBmx7xC7Dmq2N43PclV9U/niyh+4Km7oz5W0FaWdz3Op+3K0Qxz8y3z"
+        }
+        
+        const response = await LibResilientPluginConstructors
+                                .get('basic-integrity')(LR, init)
+                                .fetch('https://resilient.is/test.json');
+                                
+        assertSpyCalls(resolvingFetchSpy, 1)
+        assertSpyCall(
+            resolvingFetchSpy,
+            0,
+            {
+                args: [
+                    'https://resilient.is/test.json',
+                    {
+                        integrity: "sha384-kn5dhxz4RpBmx7xC7Dmq2N43PclV9U/niyh+4Km7oz5W0FaWdz3Op+3K0Qxz8y3z"
+                    }
+                ]
+            })
+        assertEquals(await response.json(), {test: "success"})
+    });
+    
+    it("should concatenate integrity data specified for the same effective URL twice (by absolute path, and by URL)", async () => {
+        init.integrity = {
+            "/test.json": "sha384-kn5dhxz4RpBmx7xC7Dmq2N43PclV9U/niyh+4Km7oz5W0FaWdz3Op+3K0Qxz8y3z",
+            "https://resilient.is/test.json": "sha256-Aj9x0DWq9GUL1L8HibLCMa8YLKnV7IYAfpYurqrFwiQ="
+        }
+        
+        const response = await LibResilientPluginConstructors
+                                .get('basic-integrity')(LR, init)
+                                .fetch('https://resilient.is/test.json');
+                                
+        assertSpyCalls(resolvingFetchSpy, 1)
+        assertSpyCall(
+            resolvingFetchSpy,
+            0,
+            {
+                args: [
+                    'https://resilient.is/test.json',
+                    {
+                        integrity: "sha256-Aj9x0DWq9GUL1L8HibLCMa8YLKnV7IYAfpYurqrFwiQ= sha384-kn5dhxz4RpBmx7xC7Dmq2N43PclV9U/niyh+4Km7oz5W0FaWdz3Op+3K0Qxz8y3z"
+                    }
+                ]
+            })
+        assertEquals(await response.json(), {test: "success"})
+    });
 
 })

plugins/basic-integrity/index.js

@@ -24,9 +24,20 @@
                 name: "alt-fetch"
             }],
             // integrity data for each piece of content
-            // relative URL -> integrity data (string)
+            // URL or absolute path -> integrity data (string)
+            // 
             // integrity data can contain multiple integrity hashes, space-separated, as per:
             // https://w3c.github.io/webappsec-subresource-integrity/#agility
+            // 
+            // integrity data specified for the same effective URL twice (using the whole URL,
+            // and then only the path) is concatenated
+            // 
+            // for example, these two entries:
+            // "https://example.org/some/path/index.html": "sha384-..."
+            // "/some/path/index.html": "sha512-..."
+            // 
+            // ...will result, if request URL is https://example.org/some/path/index.html and
+            // origin is https://example.org, in concatenated integrity data for that request.
             integrity: {},
             // if an URL has no integrity data associated with it, should it be allowed or not?
             requireIntegrity: true
@@ -57,10 +68,21 @@
             }
             
             // do we have integrity data in config?
+            // 
+            // the service worker would send the fetch our way only if the origin matched the URL
+            // and we only ever get full URLs anyway
+            
+            // trying a full URL (schema://example.com/...)
             if (url in config.integrity) {
                 integrity += ' ' + config.integrity[url]
             }
             
+            // trying just the path, without schema and without the domain
+            let path = url.replace(self.location.origin, "")
+            if (path in config.integrity) {
+                integrity += ' ' + config.integrity[path]
+            }
+            
             // some cleanup
             integrity = integrity.trim()
             
@@ -73,7 +95,7 @@
             } else if (config.requireIntegrity) {
                 
                 // bail if integrity data is not available
-                throw new Error(`Integrity data required but not provided for: ${url}`)
+                throw new Error(`Integrity data required but not provided for: ${url} nor ${path}`)
             }
             
             // log

plugins/fetch/index.html

@@ -0,0 +1,18 @@
+<!DOCTYPE html>
+<html>
+    <head>
+        <script>
+            self.LibResilientPluginConstructors = self.LibResilientPluginConstructors || new Map()
+            self.log = console.log
+        </script>
+        <script src="./index.js"></script>
+        <script>
+            var theplugin = LibResilientPluginConstructors.get('fetch')(self)
+        </script>
+    </head>
+    <body>
+        <h1><code>fetch()</code></h1>
+        <p>This is a simple debugging harness for the <code>fetch</code> plugin of LibResilient.</p>
+        <p>The plugin should now have been initialized in the <code>theplugin</code> global variable. Open your browser's JavaScript console to start playing with it.</p>
+    </body>
+</html>

service-worker.js

@@ -40,7 +40,15 @@ if (typeof self.LibResilientConfig !== 'object' || self.LibResilientConfig === n
             'service-worker',
             'fetch',
             'cache'
-        ]
+        ],
+        // should we normalize query params?
+        // 
+        // this usually makes sense: a request to example.com/?a=a&b=b is
+        // exactly equivalent to example.com/?b=b&a=a
+        // 
+        // but in case a given website does something weird with query params...
+        // ..normalization can be disabled here
+        normalizeQueryParams: true
     }
 }
 
@@ -90,6 +98,13 @@ let verifyConfigData = (cdata) => {
             return false;
         }
     }
+    // normalizeQueryParams is optional
+    if ("normalizeQueryParams" in cdata) {
+        if (cdata.normalizeQueryParams !== true && cdata.normalizeQueryParams !=- false) {
+            self.log('service-worker', 'fetched config contains invalid "normalizeQueryParams" data (boolean expected)')
+            return false;
+        }
+    }
     // we're good
     return true;
 }
@@ -938,6 +953,16 @@ self.addEventListener('fetch', async event => {
         // clean the URL, removing any fragment identifier
         var url = event.request.url.replace(/#.+$/, '');
         
+        // normalize query params, if we want that
+        if (self.LibResilientConfig.normalizeQueryParams) {
+            self.log('service-worker', 'normalizing query params')
+            url = url.split('?')
+            if (url.length > 1) {
+                url[1] = url[1].split('&').sort().join('&')
+            }
+            url = url.join('?')
+        }
+        
         // get the init object from Request
         var init = initFromRequest(event.request)