kopia lustrzana https://gitlab.com/rysiekpl/libresilient
Merge branch 'wip-dnslink-ipfs' into 'master'
Plugin: dnslink-ipfs See merge request rysiekpl/libresilient!15merge-requests/16/head
Michał "rysiek" Woźniak
commit
9e5fc8ff72
|
|
@ -1,13 +1,6 @@
|
|||
{
|
||||
"plugins": [{
|
||||
"name": "basic-integrity",
|
||||
"requireIntegrity": false,
|
||||
"integrity": {
|
||||
"http://localhost:8000/__tests__/test.json": "sha256-FCNALvZ0mSxEs0+SjOgx/sDFFVuh0MwkhhYnI0UJWDg="
|
||||
},
|
||||
"uses": [{
|
||||
"name": "fetch"
|
||||
}]
|
||||
"name": "dnslink-ipfs"
|
||||
}],
|
||||
"loggedComponents": ["service-worker", "fetch", "cache", "basic-integrity"]
|
||||
"loggedComponents": ["service-worker", "dnslink-ipfs"]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,206 @@
|
|||
const makeServiceWorkerEnv = require('service-worker-mock');
|
||||
|
||||
describe("plugin: dnslink-ipfs", () => {
|
||||
beforeEach(() => {
|
||||
Object.assign(global, makeServiceWorkerEnv());
|
||||
jest.resetModules();
|
||||
init = {
|
||||
name: 'dnslink-ipfs',
|
||||
gunPubkey: 'stub'
|
||||
}
|
||||
global.LibResilientPluginConstructors = new Map()
|
||||
LR = {
|
||||
log: jest.fn((component, ...items)=>{
|
||||
console.debug(component + ' :: ', ...items)
|
||||
})
|
||||
}
|
||||
|
||||
global.Ipfs = {
|
||||
ipfsFixtureAddress: 'QmiPFSiPFSiPFSiPFSiPFSiPFSiPFSiPFSiPFSiPFSiPFS',
|
||||
create: ()=>{
|
||||
return Promise.resolve({
|
||||
cat: (path)=>{
|
||||
return {
|
||||
sourceUsed: false,
|
||||
next: ()=>{
|
||||
if (path.endsWith('nonexistent.path')) {
|
||||
throw new Error('Error: file does not exist')
|
||||
}
|
||||
var prevSourceUsed = self.sourceUsed
|
||||
self.sourceUsed = true
|
||||
var val = undefined
|
||||
if (!prevSourceUsed) {
|
||||
var val = Uint8Array.from(
|
||||
Array
|
||||
.from(JSON.stringify({
|
||||
test: "success",
|
||||
path: path
|
||||
}))
|
||||
.map(
|
||||
letter => letter.charCodeAt(0)
|
||||
)
|
||||
)
|
||||
}
|
||||
return Promise.resolve({
|
||||
done: prevSourceUsed,
|
||||
value: val
|
||||
})
|
||||
}
|
||||
}
|
||||
},
|
||||
name: {
|
||||
resolve: (path)=>{
|
||||
var result = path.replace(
|
||||
'/ipns/' + self.location.origin.replace('https://', ''),
|
||||
'/ipfs/' + Ipfs.ipfsFixtureAddress
|
||||
)
|
||||
return {
|
||||
next: ()=> {
|
||||
return Promise.resolve({
|
||||
done: false,
|
||||
value: result
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
self.Ipfs = global.Ipfs
|
||||
|
||||
})
|
||||
|
||||
test("it should register in LibResilientPlugins", () => {
|
||||
require("../../../plugins/dnslink-ipfs/index.js");
|
||||
expect(LibResilientPluginConstructors.get('dnslink-ipfs')(LR, init).name).toEqual('dnslink-ipfs');
|
||||
});
|
||||
|
||||
test("IPFS setup should be initiated", async ()=>{
|
||||
self.importScripts = jest.fn()
|
||||
require("../../../plugins/dnslink-ipfs/index.js");
|
||||
try {
|
||||
await LibResilientPluginConstructors.get('dnslink-ipfs')(LR, init).fetch('/test.json')
|
||||
} catch {}
|
||||
expect(self.importScripts).toHaveBeenNthCalledWith(1, './lib/ipfs.js')
|
||||
})
|
||||
|
||||
test("fetching should error out for unpublished content", async ()=>{
|
||||
require("../../../plugins/dnslink-ipfs/index.js");
|
||||
|
||||
expect.assertions(1)
|
||||
try {
|
||||
await LibResilientPluginConstructors.get('dnslink-ipfs')(LR, init).fetch(self.location.origin + '/nonexistent.path')
|
||||
} catch(e) {
|
||||
expect(e).toEqual(new Error('Error: file does not exist'))
|
||||
}
|
||||
})
|
||||
|
||||
// TODO: probably not necessary in the long run?
|
||||
test("fetching a path ending in <path>/ should instead fetch <path>/index.html", async ()=>{
|
||||
require("../../../plugins/dnslink-ipfs/index.js");
|
||||
|
||||
try {
|
||||
var response = await LibResilientPluginConstructors.get('dnslink-ipfs')(LR, init).fetch(self.location.origin + '/test/')
|
||||
} catch(e) {
|
||||
}
|
||||
var blob = await response.blob()
|
||||
expect(JSON.parse(new TextDecoder().decode(blob.parts[0]))).toEqual({test: "success", path: "/ipfs/" + global.Ipfs.ipfsFixtureAddress + '/test/index.html'})
|
||||
})
|
||||
|
||||
test("content types should be guessed correctly when fetching", async ()=>{
|
||||
require("../../../plugins/dnslink-ipfs/index.js");
|
||||
var dnslinkIpfsPlugin = LibResilientPluginConstructors.get('dnslink-ipfs')(LR, init)
|
||||
try {
|
||||
await dnslinkIpfsPlugin.fetch(self.location.origin + '/test/')
|
||||
} catch(e) {}
|
||||
expect(LR.log).toHaveBeenCalledWith('dnslink-ipfs', " +-- guessed contentType : text/html")
|
||||
LR.log.mockClear()
|
||||
|
||||
try {
|
||||
await dnslinkIpfsPlugin.fetch(self.location.origin + '/test.htm')
|
||||
} catch(e) {}
|
||||
expect(LR.log).toHaveBeenCalledWith('dnslink-ipfs', " +-- guessed contentType : text/html")
|
||||
LR.log.mockClear()
|
||||
|
||||
try {
|
||||
await dnslinkIpfsPlugin.fetch(self.location.origin + '/test.css')
|
||||
} catch(e) {}
|
||||
expect(LR.log).toHaveBeenCalledWith('dnslink-ipfs', " +-- guessed contentType : text/css")
|
||||
LR.log.mockClear()
|
||||
|
||||
try {
|
||||
await dnslinkIpfsPlugin.fetch(self.location.origin + '/test.js')
|
||||
} catch(e) {}
|
||||
expect(LR.log).toHaveBeenCalledWith('dnslink-ipfs', " +-- guessed contentType : text/javascript")
|
||||
LR.log.mockClear()
|
||||
|
||||
try {
|
||||
await dnslinkIpfsPlugin.fetch(self.location.origin + '/test.json')
|
||||
} catch(e) {}
|
||||
expect(LR.log).toHaveBeenCalledWith('dnslink-ipfs', " +-- guessed contentType : application/json")
|
||||
LR.log.mockClear()
|
||||
|
||||
try {
|
||||
await dnslinkIpfsPlugin.fetch(self.location.origin + '/test.svg')
|
||||
} catch(e) {}
|
||||
expect(LR.log).toHaveBeenCalledWith('dnslink-ipfs', " +-- guessed contentType : image/svg+xml")
|
||||
LR.log.mockClear()
|
||||
|
||||
try {
|
||||
await dnslinkIpfsPlugin.fetch(self.location.origin + '/test.ico')
|
||||
} catch(e) {}
|
||||
expect(LR.log).toHaveBeenCalledWith('dnslink-ipfs', " +-- guessed contentType : image/x-icon")
|
||||
})
|
||||
|
||||
test("fetching should work", async ()=>{
|
||||
require("../../../plugins/dnslink-ipfs/index.js");
|
||||
|
||||
let response = await LibResilientPluginConstructors.get('dnslink-ipfs')(LR, init).fetch(self.location.origin + '/test.json')
|
||||
expect(response.body.type).toEqual('application/json')
|
||||
var blob = await response.blob()
|
||||
expect(JSON.parse(new TextDecoder().decode(blob.parts[0]))).toEqual({test: "success", path: "/ipfs/" + global.Ipfs.ipfsFixtureAddress + '/test.json'})
|
||||
})
|
||||
|
||||
test("publish() should throw an error", async ()=>{
|
||||
require("../../../plugins/dnslink-ipfs/index.js");
|
||||
|
||||
expect.assertions(1)
|
||||
try {
|
||||
LibResilientPluginConstructors.get('dnslink-ipfs')(LR, init).publish()
|
||||
} catch(e) {
|
||||
expect(e).toEqual(new Error("Not implemented yet."))
|
||||
}
|
||||
})
|
||||
|
||||
test("IPFS load error should be handled", async ()=>{
|
||||
|
||||
global.Ipfs.create = ()=>{
|
||||
throw new Error('Testing IPFS loading failure')
|
||||
}
|
||||
require("../../../plugins/dnslink-ipfs/index.js");
|
||||
|
||||
expect.assertions(1)
|
||||
try {
|
||||
await LibResilientPluginConstructors.get('dnslink-ipfs')(LR, init).fetch('/test.json')
|
||||
} catch(e) {
|
||||
expect(e).toEqual(new Error("Error: Testing IPFS loading failure"))
|
||||
}
|
||||
})
|
||||
|
||||
test("importScripts being undefined should be handled", async ()=>{
|
||||
self.importScripts = undefined
|
||||
require("../../../plugins/dnslink-ipfs/index.js");
|
||||
|
||||
try {
|
||||
await LibResilientPluginConstructors.get('dnslink-ipfs')(LR, init).fetch('/test.json')
|
||||
} catch(e) {
|
||||
}
|
||||
|
||||
expect(LR.log).toHaveBeenCalledWith("dnslink-ipfs", "Importing IPFS-related libraries...")
|
||||
expect(LR.log).toHaveBeenCalledWith("dnslink-ipfs", "assuming these scripts are already included:")
|
||||
expect(LR.log).toHaveBeenCalledWith("dnslink-ipfs", "+--", "./lib/ipfs.js")
|
||||
})
|
||||
|
||||
|
||||
});
|
||||
118
lib/ipfs.js
118
lib/ipfs.js
File diff suppressed because one or more lines are too long
|
|
@ -0,0 +1,43 @@
|
|||
# Plugin: `dnslink-ipfs`
|
||||
|
||||
- **status**: beta
|
||||
- **type**: [transport plugin](../../docs/ARCHITECTURE.md#transport-plugins)
|
||||
|
||||
## Configuration
|
||||
|
||||
No specific configuration is available.
|
||||
|
||||
## Description
|
||||
|
||||
This plugin relies on the [DNSLink standard](https://dnslink.org/) in order to resolve the IPFS [CID](https://docs.ipfs.tech/concepts/content-addressing/#identifier-formats) of the up-to-date content, and then uses [IPFS](https://docs.ipfs.tech/concepts/what-is-ipfs/#what-is-ipfs) to retrieve the content itself.
|
||||
|
||||
For this plugin to work, the website's domain needs to have the `_dnslink` DNS label with a `TXT` record pointing to the latest IPFS CID of content. [IPNS](https://docs.ipfs.tech/concepts/ipns/) DNSlinks are *not* supported currently, due to a [`js-ipfs` issue with IPNS resolution in the browser](https://github.com/ipfs/js-ipfs/issues/2921).
|
||||
|
||||
Deploying content to IPFS and updating the `_dnslink` `TXT` record is beyond the scope of this plugin, and needs to be set-up by the administrator of the website.
|
||||
|
||||
## Operation
|
||||
|
||||
IPFS is a good way of making static content available in a decentralized way. It uses content-addressing (IPFS address, or "CID", of a particular file depends on its contents), which means that a given CID will always identify a single specific file (or rather, its contents), as long as the file is available anywhere on the IPFS network.
|
||||
|
||||
IPFS CIDs can also exist for directories. In such cases, they reference CIDs of content (directories and files) contained within. Having a single CID of a directory of content hosted on IPFS is enough to be able to retrieve every file anywhere below in that directory structure.
|
||||
|
||||
However, content-addressing means that IPFS addresses are immutable. This is hardly acceptable for a website, so a strategy is needed to distribute the new IPFS CIDs each time the content changes. This is where DNSLink comes in — it specifies a standard way of distributing an up-to-date IPFS CID tied to a specific *domain name*. The IPFS CID for a DNSLink-enabled domain (like say, [`resilient.is`](https://resilient.is/)) can be found in the `TXT` record of the `_dnslink` name in it (so, [`_dnslink.resilient.is`](https://dns-lookup.com/_dnslink.resilient.is):
|
||||
|
||||
```bash
|
||||
$ kdig +short TXT _dnslink.resilient.is.
|
||||
"dnslink=/ipfs/QmPCXrKyVqeXfyZg4xhJFdQPeKqTPjJgdA65iqFvjVVKp2"
|
||||
```
|
||||
|
||||
IPFS implementations, including the [`js-ipfs`](https://js.ipfs.io/) library used by this plugin, attempt to perform DNS lookups in a way that works around any potentiall local DNS issues. This means that even if a website using the `dnslink-ipfs` plugin is temporarily unavailable locally due to name resolution failure, as long as *some* IPFS nodes can resolve the `_dnslink` label, the plugin will work.
|
||||
|
||||
Once the `_dnslink` resolution is completed, `js-ipfs` is used to retrieve the content of the requested file directly from the IPFS network.
|
||||
|
||||
## Deploying content
|
||||
|
||||
This plugin focuses only on content retrieval. For it to work, the website needs to have a way of pushing content to IPFS and updating the `_dnslink` label with the new IPFS CID.
|
||||
|
||||
The former can be achieved by running your own IPFS node (using [Kubo](https://github.com/ipfs/kubo/), for example), or using third party [IPFS pinning service](https://docs.ipfs.io/concepts/persistence/#pinning-services).
|
||||
|
||||
The latter could involve using your DNS provider's API to automatically update the relevant `TXT` record, using [DNS Dynamic Update](https://www.rfc-editor.org/rfc/rfc2136) if supported by your DNS provider, or running your own minimal DNS server and delegating `_dnslink` zone to it from your main zone — this is the strategy used for `resilient.is` currently.
|
||||
|
||||
One important consideration is the Time-to-live (`TTL`) value on the `_dnslink` `TXT` record: if content updates happen often and need to propagate fast, it needs to be as low as possible, but that will drive more DNS traffic to the nameserver, which might be a consideration. A `TTL` of `900` (15min) is probably a reasonable compromise value to start with.
|
||||
|
|
@ -0,0 +1,205 @@
|
|||
/**
|
||||
* this is the default DNSLink+IPFS strategy plugin
|
||||
* for LibResilient.
|
||||
*
|
||||
* it uses DNSLink for content address resolution
|
||||
* and IPFS for delivery
|
||||
*/
|
||||
|
||||
|
||||
/* ========================================================================= *\
|
||||
|* === General stuff and setup === *|
|
||||
\* ========================================================================= */
|
||||
|
||||
// no polluting of the global namespace please
|
||||
(function(LRPC){
|
||||
// this never changes
|
||||
const pluginName = "dnslink-ipfs"
|
||||
LRPC.set(pluginName, (LR, init={})=>{
|
||||
|
||||
var ipfsPromise;
|
||||
|
||||
// sane defaults
|
||||
let defaultConfig = {
|
||||
// the IPFS gateway we're using for verification when publishing; default is usually ok
|
||||
//ipfsGateway: 'https://gateway.ipfs.io'
|
||||
}
|
||||
|
||||
// merge the defaults with settings from init
|
||||
let config = {...defaultConfig, ...init}
|
||||
|
||||
/**
|
||||
* importing stuff works differently between a browser window context
|
||||
* and a ServiceWorker context, because things just can't be easy and sane
|
||||
*/
|
||||
function doImport() {
|
||||
var args = Array.prototype.slice.call(arguments);
|
||||
if (typeof self.importScripts !== 'undefined') {
|
||||
self.importScripts.apply(self, args)
|
||||
} else {
|
||||
LR.log(pluginName, 'assuming these scripts are already included:')
|
||||
args.forEach(function(src){
|
||||
LR.log(pluginName, '+--', src)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
async function setup_ipfs() {
|
||||
LR.log(pluginName, 'Importing IPFS-related libraries...');
|
||||
doImport(
|
||||
"./lib/ipfs.js");
|
||||
LR.log(pluginName, 'Setting up IPFS...')
|
||||
try {
|
||||
var ipfs = await self.Ipfs.create();
|
||||
LR.log(pluginName, '+-- IPFS loaded :: ipfs is : ' + typeof ipfs)
|
||||
return ipfs
|
||||
} catch(e) {
|
||||
LR.log(pluginName, '+-- Error loading IPFS: ' + e)
|
||||
throw new Error(e)
|
||||
}
|
||||
}
|
||||
|
||||
/* ========================================================================= *\
|
||||
|* === Main functionality === *|
|
||||
\* ========================================================================= */
|
||||
|
||||
/**
|
||||
* the workhorse of this plugin
|
||||
*/
|
||||
async function getContentFromDNSLinkAndIPFS(url, init={}) {
|
||||
|
||||
// make sure IPFS is set-up
|
||||
var ipfs = await ipfsPromise
|
||||
|
||||
// we don't want the scheme
|
||||
var dnslinkAddr = url.replace(/https?:\/\//, '')
|
||||
|
||||
/*
|
||||
* if the dnslinkAddr ends in '/', append 'index.html' to it
|
||||
*
|
||||
* TODO: might not be necessary; if removed, update the content-type switch statement below!
|
||||
*/
|
||||
if (dnslinkAddr.charAt(dnslinkAddr.length - 1) === '/') {
|
||||
LR.log(pluginName, "NOTICE: address ends in '/', assuming '/index.html' should be appended.");
|
||||
dnslinkAddr += 'index.html';
|
||||
}
|
||||
|
||||
LR.log(pluginName, "+-- starting DNSLink lookup of: '" + dnslinkAddr + "'");
|
||||
|
||||
/*
|
||||
* naïvely assume content type based on file extension
|
||||
* TODO: this needs a fix
|
||||
*/
|
||||
var contentType = '';
|
||||
switch (dnslinkAddr.split('.').pop().toLowerCase()) {
|
||||
case 'html':
|
||||
case 'htm':
|
||||
contentType = 'text/html';
|
||||
break;
|
||||
case 'css':
|
||||
contentType = 'text/css';
|
||||
break;
|
||||
case 'js':
|
||||
contentType = 'text/javascript';
|
||||
break;
|
||||
case 'svg':
|
||||
contentType = 'image/svg+xml';
|
||||
break;
|
||||
case 'ico':
|
||||
contentType = 'image/x-icon';
|
||||
break;
|
||||
case 'json':
|
||||
contentType = 'application/json';
|
||||
break;
|
||||
}
|
||||
LR.log(pluginName, " +-- guessed contentType : " + contentType);
|
||||
|
||||
// TODO: error handling!
|
||||
return ipfs.name.resolve('/ipns/' + dnslinkAddr).next().then(ipfsaddr => {
|
||||
|
||||
// TODO: use the iterator, luke
|
||||
LR.log(pluginName, "+-- starting IPFS retrieval of: '" + ipfsaddr.value + "'");
|
||||
return ipfs.cat(ipfsaddr.value);
|
||||
|
||||
}).then(async (source) => {
|
||||
|
||||
LR.log(pluginName, '+-- started receiving file data')
|
||||
// source is an iterator
|
||||
// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Iterators_and_Generators
|
||||
var filedata = await source.next();
|
||||
|
||||
// did we get anything?
|
||||
if (filedata.value) {
|
||||
|
||||
// initialize
|
||||
var content = new Uint8Array()
|
||||
|
||||
do {
|
||||
LR.log(pluginName, ' +-- new data:', filedata.done, filedata.value.length)
|
||||
var newContent = new Uint8Array(content.length + filedata.value.length);
|
||||
newContent.set(content)
|
||||
newContent.set(filedata.value, content.length)
|
||||
content = newContent
|
||||
filedata = await source.next()
|
||||
} while (! filedata.done)
|
||||
|
||||
LR.log(pluginName, '+-- got a DNSLink-resolved IPFS-stored file; content is: ' + typeof content);
|
||||
|
||||
// creating and populating the blob
|
||||
var blob = new Blob(
|
||||
[content],
|
||||
{'type': contentType}
|
||||
);
|
||||
|
||||
return new Response(
|
||||
blob,
|
||||
{
|
||||
'status': 200,
|
||||
'statusText': 'OK',
|
||||
'headers': {
|
||||
'Content-Type': contentType,
|
||||
'ETag': 'WOLOLO',
|
||||
'X-LibResilient-Method': pluginName,
|
||||
'X-LibResilient-ETag': 'WOLOLO'
|
||||
}
|
||||
}
|
||||
);
|
||||
} else {
|
||||
LR.log(pluginName, '+-- IPFS retrieval failed: no content.')
|
||||
throw new Error('IPFS retrieval failed: no content.')
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
/* ========================================================================= *\
|
||||
|* === Publishing stuff === *|
|
||||
\* ========================================================================= */
|
||||
/*
|
||||
* TODO: to be implemented
|
||||
*/
|
||||
let publishContent = (resource, user, password) => {
|
||||
throw new Error("Not implemented yet.")
|
||||
}
|
||||
|
||||
/* ========================================================================= *\
|
||||
|* === Initialization === *|
|
||||
\* ========================================================================= */
|
||||
|
||||
// we probably need to handle this better
|
||||
ipfsPromise = setup_ipfs();
|
||||
|
||||
|
||||
// and add ourselves to it
|
||||
// with some additional metadata
|
||||
return {
|
||||
name: pluginName,
|
||||
description: 'Decentralized resource fetching using DNSLink for content address resolution and IPFS for content delivery.',
|
||||
version: 'COMMIT_UNKNOWN',
|
||||
fetch: getContentFromDNSLinkAndIPFS,
|
||||
publish: publishContent
|
||||
}
|
||||
|
||||
})
|
||||
// done with not polluting the global namespace
|
||||
})(LibResilientPluginConstructors)
|
||||
Ładowanie…
Reference in New Issue