From bd8d56efaf972d0a8dee20e23d0f8b55f08ef267 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27rysiek=27=20Wo=C5=BAniak?= Date: Sat, 26 Nov 2022 18:40:21 +0000 Subject: [PATCH 01/38] started working on lrcli and on basic-integrity plugin cli (ref. %5) --- cli/README.md | 6 +++ cli/lrcli.js | 90 ++++++++++++++++++++++++++++++++++ plugins/basic-integrity/cli.js | 58 ++++++++++++++++++++++ 3 files changed, 154 insertions(+) create mode 100644 cli/README.md create mode 100644 cli/lrcli.js create mode 100644 plugins/basic-integrity/cli.js diff --git a/cli/README.md b/cli/README.md new file mode 100644 index 0000000..35fbc3d --- /dev/null +++ b/cli/README.md @@ -0,0 +1,6 @@ +# LibResilient CLI + +This is the command-line interface for LibResilient. It is supposed to understand the `config.json` configuration file, and simplify deployment of content so as to ensure that all configured transport plugins work as expected. It should also *test* if those channels work as expected. + +This is a work in progress and not production-ready yet. + diff --git a/cli/lrcli.js b/cli/lrcli.js new file mode 100644 index 0000000..4066e50 --- /dev/null +++ b/cli/lrcli.js @@ -0,0 +1,90 @@ +import { parse } from "https://deno.land/std/flags/mod.ts"; + +var parsed_args = parse( + Deno.args, + { + default: { + h: false, + }, + boolean: [ "h" ], + string: [], + alias: { + h: [ "help", "usage", "u" ] + }, + negatable: [], + // a function which is invoked with a command line parameter not defined + // in the options configuration object. If the function returns false, + // the unknown option is not added to parsedArgs. + unknown: null + } +); + +let printUsage = () => { + let usage = ` +Command-line interface for LibResilient. + +This script creates a common interface to CLI actions implemented by LibResilient plugins. + +Usage: + ${new URL('', import.meta.url).toString().split('/').at(-1)} [options] [plugin-name [plugin-options]] + +Options: + + -h, --help [plugin-name] + -u, --usage [plugin-name] + Print this message, if no plugin-name is given. + If plugin-name is provided, print usage information of that plugin. + +` + + console.log(usage) +} + +let printpluginUsage = (plugin) => { + let usage = ` +LibResilient CLI for plugin: ${plugin.name}. + +Plugin Description: + ${plugin.description} + +Usage: + ${new URL('', import.meta.url).toString().split('/').at(-1)} [general-options] ${plugin.name} [plugin-action [action-options]] + +General Options: + + -h, --help [plugin-name] + -u, --usage [plugin-name] + Print this message, if no plugin-name is given. + If plugin-name is provided, print usage information of that plugin. + +Actions and Action Options: +` + + console.log(usage) +} + +// assuming: +// - the first unknown argument is the name of the plugin +// - plugins live in ../plugins//cli.js, relative to lrcli.js location +// - only one plugin loaded per invocation, at least for now +let plugin +if (parsed_args._.length > 0) { + plugin = await import(`../plugins/${parsed_args._[0]}/cli.js`); + if (parsed_args._.length > 1) { + let action = parsed_args._[1] + if (action in plugin.actions) { + plugin.actions[action].run() + } else { + printpluginUsage(plugin) + } + } else { + printpluginUsage(plugin) + } +} else { + printUsage() +} + + + +console.log(parsed_args) +console.log(new URL('', import.meta.url).toString().split('/').at(-1)) diff --git a/plugins/basic-integrity/cli.js b/plugins/basic-integrity/cli.js new file mode 100644 index 0000000..de0356a --- /dev/null +++ b/plugins/basic-integrity/cli.js @@ -0,0 +1,58 @@ +/* ========================================================================= *\ +|* === basic-integrity: pre-configured subresource integrity for content === *| +\* ========================================================================= */ + +/** + * basic-integrity plugin's deploy/utility functions + * + * this code expects a Deno runtime: + * https://deno.land/ + */ + +/** + * get integrity data for specified urls, using specified algorithms + * + * naming of algorithms as accepted by SubtleCrypto.digest(): + * https://developer.mozilla.org/en-US/docs/Web/API/SubtleCrypto/digest + * + * paths - array of strings, paths to individual pieces of content + * algos - array of algorithms to use to calculate digests (default: "SHA-256") + */ +let getIntegrity = (paths, algos=["SHA-256"]) => { + + // we need non-emtpy arrays of string in the arguments + if (!Array.isArray(paths) || (paths.length == 0)) { + throw new Error("Expected non-empty array of strings in the 'paths' argument.") + } + if (!Array.isArray(algos) || (algos.length == 0)) { + throw new Error("Expected non-empty array of strings in the 'algos' argument.") + } + + console.log('weeee') + console.log(paths) + console.log(algos) + +} + + +// this never changes +const pluginName = "basic-integrity" +const pluginDescription = "verifying subresource integrity for resources fetched by other plugins" +const pluginVersion = 'COMMIT_UNKNOWN' +const pluginActions = { + "get-integrity": { + run: getIntegrity, + description: "calculate subresource integrity hashes for provided files", + arguments: { + paths: "array of strings, paths to individual pieces of content", + algos: "array of SubtleCrypto.digest-compatible algorithm names nto use to calculate digests (default: \"SHA-256\")" + } + } +} + +export { + pluginName as name, + pluginDescription as description, + pluginVersion as version, + pluginActions as actions +} From e07c7fe84a0c97d211698cf5dc845bcd440edfc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27rysiek=27=20Wo=C5=BAniak?= Date: Mon, 5 Dec 2022 19:09:37 +0000 Subject: [PATCH 02/38] bugfix in docs/ARCHITECTURE.md --- docs/ARCHITECTURE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index f0d3111..4b284d4 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -114,7 +114,7 @@ A more robust configuration could look like this: "name": "alt-fetch", "endpoints": [ "https://fallback-endpoint.example.com" - ]} + ] }] } ``` From e57877b4c89bc3ddcd4d7253c393e37c86786e27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27rysiek=27=20Wo=C5=BAniak?= Date: Mon, 5 Dec 2022 21:34:40 +0000 Subject: [PATCH 03/38] alt-fetch README fix --- plugins/alt-fetch/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/alt-fetch/README.md b/plugins/alt-fetch/README.md index a482d58..0027304 100644 --- a/plugins/alt-fetch/README.md +++ b/plugins/alt-fetch/README.md @@ -5,7 +5,7 @@ This transport plugin uses standard [`fetch()`](https://developer.mozilla.org/en-US/docs/Web/API/fetch) to retrieve remote content from alternative endpoints — that is, HTTPS endpoints that are not in the original domain. This enables retrieving content even if the website on the original domain is down for whatever reason. The list of alternative endpoints is configured via LibResilient config file, `config.json`. -Compare: [`dnslink-fetch`](../alt-fetch/). +Compare: [`dnslink-fetch`](../dnslink-fetch/). As per LibResilient architecture, this plugin adds `X-LibResilient-Method` and `X-LibResilient-ETag` headers to the returned response. From 49de3f07ec58fd0a717b620aa9fed95b7dd44708 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27rysiek=27=20Wo=C5=BAniak?= Date: Wed, 7 Dec 2022 00:13:52 +0000 Subject: [PATCH 04/38] implemented argument passing to plugin's cli modules --- cli/lrcli.js | 63 +++++++++++++++++++++++++++++++--- plugins/basic-integrity/cli.js | 46 +++++++++++++++++++++---- 2 files changed, 98 insertions(+), 11 deletions(-) diff --git a/cli/lrcli.js b/cli/lrcli.js index 4066e50..879a76b 100644 --- a/cli/lrcli.js +++ b/cli/lrcli.js @@ -6,11 +6,13 @@ var parsed_args = parse( default: { h: false, }, + stopEarly: true, boolean: [ "h" ], string: [], alias: { h: [ "help", "usage", "u" ] }, + collect: [], negatable: [], // a function which is invoked with a command line parameter not defined // in the options configuration object. If the function returns false, @@ -63,17 +65,73 @@ Actions and Action Options: console.log(usage) } +let parsePluginActionArgs = (args, argdef) => { + + var plugin_args_config = { + boolean: [], + string: [], + alias: {}, + collect: [], + negatable: [], + unknown: null, + default: {}, + alias: {} + } + + for (const [argname, argconfig] of Object.entries(argdef)) { + if ( ("collect" in argconfig) && (argconfig.collect === true) ) { + plugin_args_config.collect.push(argname) + } + if ( ("string" in argconfig) && (argconfig.string === true) ) { + plugin_args_config.string.push(argname) + } + if ( ("boolean" in argconfig) && (argconfig.boolean === true) ) { + plugin_args_config.boolean.push(argname) + } + if ( ("negatable" in argconfig) && (argconfig.negatable === true) ) { + plugin_args_config.negatable.push(argname) + } + if ("default" in argconfig) { + plugin_args_config.default[argname] = argconfig.default + } + } + + var parsed = parse(args, plugin_args_config) + console.log(parsed) + + var result = [] + for (const argname of Object.keys(argdef)) { + if (argname in parsed) { + result.push(parsed[argname]) + } + } + + return result +} + // assuming: // - the first unknown argument is the name of the plugin // - plugins live in ../plugins//cli.js, relative to lrcli.js location // - only one plugin loaded per invocation, at least for now +// +// we *always* pass arguments to plugins as arrays of strings, +// even if we only got one value let plugin if (parsed_args._.length > 0) { plugin = await import(`../plugins/${parsed_args._[0]}/cli.js`); if (parsed_args._.length > 1) { + let action = parsed_args._[1] if (action in plugin.actions) { - plugin.actions[action].run() + + var parsed_plugin_args = parsePluginActionArgs( + // removing the plugin name and the method name + parsed_args._.slice(2), + plugin.actions[action].arguments + ) + + plugin.actions[action].run(...parsed_plugin_args) + } else { printpluginUsage(plugin) } @@ -84,7 +142,4 @@ if (parsed_args._.length > 0) { printUsage() } - - -console.log(parsed_args) console.log(new URL('', import.meta.url).toString().split('/').at(-1)) diff --git a/plugins/basic-integrity/cli.js b/plugins/basic-integrity/cli.js index de0356a..559e1d7 100644 --- a/plugins/basic-integrity/cli.js +++ b/plugins/basic-integrity/cli.js @@ -9,6 +9,29 @@ * https://deno.land/ */ +let getFileIntegrity = async (path, algos) => { + + const file = await Deno.open( + path, + { read: true } + ); + const fileInfo = await file.stat(); + console.log(fileInfo) + if (fileInfo.isFile) { + const decoder = new TextDecoder(); + for await (const chunk of file.readable) { + console.log(decoder.decode(chunk)); + } + } + // putting this in a try-catch block as the file + // is apparently being auto-closed? + // https://issueantenna.com/repo/denoland/deno/issues/15442 + try { + await file.close(); + } catch (BadResource) {} +} + + /** * get integrity data for specified urls, using specified algorithms * @@ -27,11 +50,11 @@ let getIntegrity = (paths, algos=["SHA-256"]) => { if (!Array.isArray(algos) || (algos.length == 0)) { throw new Error("Expected non-empty array of strings in the 'algos' argument.") } - - console.log('weeee') - console.log(paths) - console.log(algos) - + + var result = paths.map(p => { + return getFileIntegrity(p, algos) + }) + console.log(result) } @@ -44,8 +67,17 @@ const pluginActions = { run: getIntegrity, description: "calculate subresource integrity hashes for provided files", arguments: { - paths: "array of strings, paths to individual pieces of content", - algos: "array of SubtleCrypto.digest-compatible algorithm names nto use to calculate digests (default: \"SHA-256\")" + path: { + description: "array of strings, paths to individual pieces of content", + collect: true, + string: true + }, + algorithm: { + description: "array of SubtleCrypto.digest-compatible algorithm names to use to calculate digests (default: \"SHA-256\")", + collect: true, + string: true, + default: "SHA-256" + } } } } From aef901fd73ad3d87832fbb407553093042ae020b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27rysiek=27=20Wo=C5=BAniak?= Date: Thu, 8 Dec 2022 13:03:06 +0000 Subject: [PATCH 05/38] basic-integrity: cli digest kinda working --- plugins/basic-integrity/cli.js | 55 +++++++++++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 4 deletions(-) diff --git a/plugins/basic-integrity/cli.js b/plugins/basic-integrity/cli.js index 559e1d7..860a5b4 100644 --- a/plugins/basic-integrity/cli.js +++ b/plugins/basic-integrity/cli.js @@ -9,6 +9,23 @@ * https://deno.land/ */ + +/** + * helper function, converting binary to base64 + * this need not be extremely fast, since it will only be used on digests + * + * binary_data - data to convert to base64 + */ +let binToBase64 = (binary_data) => { + return btoa( + (new Uint8Array(binary_data)) + .reduce((bin, byte)=>{ + return bin += String.fromCharCode(byte) + }, '') + ) +} + + let getFileIntegrity = async (path, algos) => { const file = await Deno.open( @@ -16,12 +33,42 @@ let getFileIntegrity = async (path, algos) => { { read: true } ); const fileInfo = await file.stat(); - console.log(fileInfo) + + // are we working with a file? if (fileInfo.isFile) { - const decoder = new TextDecoder(); - for await (const chunk of file.readable) { - console.log(decoder.decode(chunk)); + console.log(`+-- reading: ${path}`) + + // initialize + var content = new Uint8Array() + var buf = new Uint8Array(1000); + + // read the first batch + var numread = file.readSync(buf); + + // read the rest, if there is anything to read + while (numread !== null) { + console.log(` +-- read: ${numread}`) + console.log(` +-- length: ${content.length}`) + + // there has to be a better way... + var new_content = new Uint8Array(content.length + numread); + console.log(` +-- new length: ${new_content.length}`) + new_content.set(content) + if (buf.length === numread) { + new_content.set(buf, content.length) + } else { + new_content.set(buf.slice(0, numread), content.length) + } + content = new_content + + // read some more + numread = file.readSync(buf); } + console.log(' +-- done.') + + var digest = binToBase64(await crypto.subtle.digest(algos[0], content)) + console.log(digest) + } // putting this in a try-catch block as the file // is apparently being auto-closed? From 3dd94f1c7574d4f4ab6247d9d31cd23274c44294 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27rysiek=27=20Wo=C5=BAniak?= Date: Thu, 8 Dec 2022 13:57:05 +0000 Subject: [PATCH 06/38] cli: we have sane output for basic-integrity plugin --- cli/lrcli.js | 12 ++++++-- plugins/basic-integrity/cli.js | 56 ++++++++++++++++++++++++++-------- 2 files changed, 52 insertions(+), 16 deletions(-) diff --git a/cli/lrcli.js b/cli/lrcli.js index 879a76b..a89abdf 100644 --- a/cli/lrcli.js +++ b/cli/lrcli.js @@ -97,7 +97,7 @@ let parsePluginActionArgs = (args, argdef) => { } var parsed = parse(args, plugin_args_config) - console.log(parsed) + //console.log(parsed) var result = [] for (const argname of Object.keys(argdef)) { @@ -130,7 +130,13 @@ if (parsed_args._.length > 0) { plugin.actions[action].arguments ) - plugin.actions[action].run(...parsed_plugin_args) + // not using console.log here because we want the *exact* output + // without any extra ending newlines + await Deno.stdout.write( + new TextEncoder().encode( + await plugin.actions[action].run(...parsed_plugin_args) + ) + ) } else { printpluginUsage(plugin) @@ -142,4 +148,4 @@ if (parsed_args._.length > 0) { printUsage() } -console.log(new URL('', import.meta.url).toString().split('/').at(-1)) +//console.log(new URL('', import.meta.url).toString().split('/').at(-1)) diff --git a/plugins/basic-integrity/cli.js b/plugins/basic-integrity/cli.js index 860a5b4..952d603 100644 --- a/plugins/basic-integrity/cli.js +++ b/plugins/basic-integrity/cli.js @@ -34,9 +34,11 @@ let getFileIntegrity = async (path, algos) => { ); const fileInfo = await file.stat(); + var result = [] + // are we working with a file? if (fileInfo.isFile) { - console.log(`+-- reading: ${path}`) + //console.log(`+-- reading: ${path}`) // initialize var content = new Uint8Array() @@ -47,12 +49,12 @@ let getFileIntegrity = async (path, algos) => { // read the rest, if there is anything to read while (numread !== null) { - console.log(` +-- read: ${numread}`) - console.log(` +-- length: ${content.length}`) + //console.log(` +-- read: ${numread}`) + //console.log(` +-- length: ${content.length}`) // there has to be a better way... var new_content = new Uint8Array(content.length + numread); - console.log(` +-- new length: ${new_content.length}`) + //console.log(` +-- new length: ${new_content.length}`) new_content.set(content) if (buf.length === numread) { new_content.set(buf, content.length) @@ -64,10 +66,16 @@ let getFileIntegrity = async (path, algos) => { // read some more numread = file.readSync(buf); } - console.log(' +-- done.') + //console.log(' +-- done.') - var digest = binToBase64(await crypto.subtle.digest(algos[0], content)) - console.log(digest) + //console.log('+-- calculating digests') + for (const algo of algos) { + //console.log(` +-- ${algo}`) + var digest = algo.toLowerCase().replace('-', '') + '-' + binToBase64(await crypto.subtle.digest(algo, content)) + //console.log(digest) + result.push(digest) + } + //console.log(`+-- file done: ${path}`) } // putting this in a try-catch block as the file @@ -76,6 +84,9 @@ let getFileIntegrity = async (path, algos) => { try { await file.close(); } catch (BadResource) {} + + // return the result + return result } @@ -88,7 +99,7 @@ let getFileIntegrity = async (path, algos) => { * paths - array of strings, paths to individual pieces of content * algos - array of algorithms to use to calculate digests (default: "SHA-256") */ -let getIntegrity = (paths, algos=["SHA-256"]) => { +let getIntegrity = async (paths, algos=["SHA-256"], output="json") => { // we need non-emtpy arrays of string in the arguments if (!Array.isArray(paths) || (paths.length == 0)) { @@ -97,11 +108,24 @@ let getIntegrity = (paths, algos=["SHA-256"]) => { if (!Array.isArray(algos) || (algos.length == 0)) { throw new Error("Expected non-empty array of strings in the 'algos' argument.") } - - var result = paths.map(p => { - return getFileIntegrity(p, algos) - }) - console.log(result) + if (!['json', 'text'].includes(output)) { + throw new Error("Expected either 'json' or 'text' in the 'output' argument.") + } + + var result = {} + for (const p of paths) { + result[p] = await getFileIntegrity(p, algos) + } + + if (output == 'json') { + return JSON.stringify(result) + } else { + var text_result = '' + for (const p of paths) { + text_result += `${p}: ${result[p].join(' ')}\n` + } + return text_result + } } @@ -124,6 +148,12 @@ const pluginActions = { collect: true, string: true, default: "SHA-256" + }, + output: { + description: "a string, defining output mode ('json' or 'text'; 'json' is default)", + collect: false, + string: true, + default: "json" } } } From b9d7d9f0eb32ee2a13c2b23fe684996c6edac8f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27rysiek=27=20Wo=C5=BAniak?= Date: Fri, 9 Dec 2022 13:45:05 +0000 Subject: [PATCH 07/38] cli: we can support multiple files passed via glob expressions; basic-integrity cli uses that now --- cli/lrcli.js | 3 +++ plugins/basic-integrity/cli.js | 25 +++++++++++++++++-------- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/cli/lrcli.js b/cli/lrcli.js index a89abdf..271d763 100644 --- a/cli/lrcli.js +++ b/cli/lrcli.js @@ -79,6 +79,9 @@ let parsePluginActionArgs = (args, argdef) => { } for (const [argname, argconfig] of Object.entries(argdef)) { + if (argname == '_') { + continue; + } if ( ("collect" in argconfig) && (argconfig.collect === true) ) { plugin_args_config.collect.push(argname) } diff --git a/plugins/basic-integrity/cli.js b/plugins/basic-integrity/cli.js index 952d603..02d704d 100644 --- a/plugins/basic-integrity/cli.js +++ b/plugins/basic-integrity/cli.js @@ -27,17 +27,19 @@ let binToBase64 = (binary_data) => { let getFileIntegrity = async (path, algos) => { - + + var result = [] + + // open the file and get some info on it const file = await Deno.open( path, { read: true } ); const fileInfo = await file.stat(); - var result = [] - // are we working with a file? if (fileInfo.isFile) { + //console.log(`+-- reading: ${path}`) // initialize @@ -77,7 +79,11 @@ let getFileIntegrity = async (path, algos) => { } //console.log(`+-- file done: ${path}`) + // we are not working with a file + } else { + result = false; } + // putting this in a try-catch block as the file // is apparently being auto-closed? // https://issueantenna.com/repo/denoland/deno/issues/15442 @@ -114,7 +120,12 @@ let getIntegrity = async (paths, algos=["SHA-256"], output="json") => { var result = {} for (const p of paths) { - result[p] = await getFileIntegrity(p, algos) + // filter-out stuff we are not interested in + // like directories etc + var r = await getFileIntegrity(p, algos) + if (r !== false) { + result[p] = r + } } if (output == 'json') { @@ -138,10 +149,8 @@ const pluginActions = { run: getIntegrity, description: "calculate subresource integrity hashes for provided files", arguments: { - path: { - description: "array of strings, paths to individual pieces of content", - collect: true, - string: true + _: { + description: "paths to individual pieces of content" }, algorithm: { description: "array of SubtleCrypto.digest-compatible algorithm names to use to calculate digests (default: \"SHA-256\")", From 17a20ba5599fe18cab414363fd0924eefd420d9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27rysiek=27=20Wo=C5=BAniak?= Date: Fri, 9 Dec 2022 18:22:24 +0000 Subject: [PATCH 08/38] cli: cleaned up the implementation a bit; improved basic-integrity descriptions --- cli/lrcli.js | 107 ++++++++++++++++++++++----------- plugins/basic-integrity/cli.js | 8 +-- 2 files changed, 75 insertions(+), 40 deletions(-) diff --git a/cli/lrcli.js b/cli/lrcli.js index 271d763..eac3cd5 100644 --- a/cli/lrcli.js +++ b/cli/lrcli.js @@ -10,7 +10,7 @@ var parsed_args = parse( boolean: [ "h" ], string: [], alias: { - h: [ "help", "usage", "u" ] + h: [ "help" ] }, collect: [], negatable: [], @@ -33,7 +33,6 @@ Usage: Options: -h, --help [plugin-name] - -u, --usage [plugin-name] Print this message, if no plugin-name is given. If plugin-name is provided, print usage information of that plugin. @@ -44,10 +43,11 @@ Options: let printpluginUsage = (plugin) => { let usage = ` -LibResilient CLI for plugin: ${plugin.name}. +CLI plugin: + ${plugin.name} Plugin Description: - ${plugin.description} + ${plugin.description.replace('\n', '\n ')} Usage: ${new URL('', import.meta.url).toString().split('/').at(-1)} [general-options] ${plugin.name} [plugin-action [action-options]] @@ -55,7 +55,6 @@ Usage: General Options: -h, --help [plugin-name] - -u, --usage [plugin-name] Print this message, if no plugin-name is given. If plugin-name is provided, print usage information of that plugin. @@ -65,6 +64,8 @@ Actions and Action Options: console.log(usage) } +//console.log(parsed_args) + let parsePluginActionArgs = (args, argdef) => { var plugin_args_config = { @@ -74,8 +75,8 @@ let parsePluginActionArgs = (args, argdef) => { collect: [], negatable: [], unknown: null, - default: {}, - alias: {} + default: { + } } for (const [argname, argconfig] of Object.entries(argdef)) { @@ -99,16 +100,21 @@ let parsePluginActionArgs = (args, argdef) => { } } + //console.log(plugin_args_config) var parsed = parse(args, plugin_args_config) //console.log(parsed) var result = [] + + // we want to keep the order of arguments + // as defined in the plugin cli code for (const argname of Object.keys(argdef)) { if (argname in parsed) { result.push(parsed[argname]) } } + // we're done return result } @@ -119,36 +125,65 @@ let parsePluginActionArgs = (args, argdef) => { // // we *always* pass arguments to plugins as arrays of strings, // even if we only got one value -let plugin -if (parsed_args._.length > 0) { - plugin = await import(`../plugins/${parsed_args._[0]}/cli.js`); - if (parsed_args._.length > 1) { - - let action = parsed_args._[1] - if (action in plugin.actions) { - - var parsed_plugin_args = parsePluginActionArgs( - // removing the plugin name and the method name - parsed_args._.slice(2), - plugin.actions[action].arguments - ) - - // not using console.log here because we want the *exact* output - // without any extra ending newlines - await Deno.stdout.write( - new TextEncoder().encode( - await plugin.actions[action].run(...parsed_plugin_args) - ) - ) - - } else { - printpluginUsage(plugin) - } - } else { - printpluginUsage(plugin) - } -} else { + +// no unknown parsed args? that means we have no plugin specified +if (parsed_args._.length == 0) { printUsage() + Deno.exit(1) } +// try loading the plugin +let plugin +try { + plugin = await import(`../plugins/${parsed_args._[0]}/cli.js`); +} catch (e) { + // unable to load the plugin? bail with info + console.log(`\n*** ${e} ***`) + printUsage() + Deno.exit(2) +} + +// if we only had exactly one unknown arg, we only have the plugin name +// but no info from the user what to do with it +// → print plugin usage and exit +if (parsed_args._.length == 1) { + console.log('\n*** No action specified for plugin ***') + printpluginUsage(plugin) + Deno.exit(3) +} + +let action = parsed_args._[1] +if ( ! (action in plugin.actions) ) { + var exit_code = 4 + if (!['--help', '-h'].includes(action)) { + console.log(`\n*** Action not supported: ${action} ***`) + exit_code = 0 + } + printpluginUsage(plugin) + Deno.exit(exit_code) +} + +var parsed_plugin_args = parsePluginActionArgs( + // removing the plugin name and the method name + parsed_args._.slice(2), + plugin.actions[action].arguments + ) + +//console.log(parsed_plugin_args) + +// not using console.log here because we want the *exact* output +// without any extra ending newlines +try { + await Deno.stdout.write( + new TextEncoder().encode( + await plugin.actions[action].run(...parsed_plugin_args) + ) + ) +} catch (e) { + console.log(`\n*** ${e} ***`) + printpluginUsage(plugin) +} + + + //console.log(new URL('', import.meta.url).toString().split('/').at(-1)) diff --git a/plugins/basic-integrity/cli.js b/plugins/basic-integrity/cli.js index 02d704d..b59e6b8 100644 --- a/plugins/basic-integrity/cli.js +++ b/plugins/basic-integrity/cli.js @@ -109,13 +109,13 @@ let getIntegrity = async (paths, algos=["SHA-256"], output="json") => { // we need non-emtpy arrays of string in the arguments if (!Array.isArray(paths) || (paths.length == 0)) { - throw new Error("Expected non-empty array of strings in the 'paths' argument.") + throw new Error("Expected non-empty list of files to generate digests of.") } if (!Array.isArray(algos) || (algos.length == 0)) { - throw new Error("Expected non-empty array of strings in the 'algos' argument.") + throw new Error("Expected non-empty list of algorithms to use.") } if (!['json', 'text'].includes(output)) { - throw new Error("Expected either 'json' or 'text' in the 'output' argument.") + throw new Error("Expected either 'json' or 'text' as output type to generate.") } var result = {} @@ -142,7 +142,7 @@ let getIntegrity = async (paths, algos=["SHA-256"], output="json") => { // this never changes const pluginName = "basic-integrity" -const pluginDescription = "verifying subresource integrity for resources fetched by other plugins" +const pluginDescription = "Verifying subresource integrity for resources fetched by other plugins.\nCLI used to generate subresource integrity hashes for provided files." const pluginVersion = 'COMMIT_UNKNOWN' const pluginActions = { "get-integrity": { From 3ecf637167b61afe944c6a8c04aaa4d59b6571fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27rysiek=27=20Wo=C5=BAniak?= Date: Fri, 9 Dec 2022 18:29:58 +0000 Subject: [PATCH 09/38] cli: shebang! --- cli/lrcli.js | 2 ++ 1 file changed, 2 insertions(+) mode change 100644 => 100755 cli/lrcli.js diff --git a/cli/lrcli.js b/cli/lrcli.js old mode 100644 new mode 100755 index eac3cd5..92840c3 --- a/cli/lrcli.js +++ b/cli/lrcli.js @@ -1,3 +1,5 @@ +#!/usr/bin/env -S deno run --allow-read + import { parse } from "https://deno.land/std/flags/mod.ts"; var parsed_args = parse( From 49b494f858054da6382d27089a8a36ba3c8533d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27rysiek=27=20Wo=C5=BAniak?= Date: Sat, 10 Dec 2022 18:35:01 +0000 Subject: [PATCH 10/38] cli: describing plugin actions along with arguments and options --- cli/lrcli.js | 53 +++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 50 insertions(+), 3 deletions(-) diff --git a/cli/lrcli.js b/cli/lrcli.js index 92840c3..665d345 100755 --- a/cli/lrcli.js +++ b/cli/lrcli.js @@ -43,6 +43,44 @@ Options: console.log(usage) } + +let printPluginActionUsage = (action, action_name) => { + + let options = "" + for (const opt in action.arguments) { + if (opt == '_') { + continue + } + options += `\n --${opt}` + if ("default" in action.arguments[opt]) { + options += ` (default: ${action.arguments[opt].default})` + } + options += `\n ${action.arguments[opt].description}` + options += `\n` + } + + let positional = '' + let positional_desc = '' + if ('_' in action.arguments) { + if ('name' in action.arguments._) { + positional = ` <${action.arguments._.name}...>` + } else { + positional = ' ' + } + positional_desc = ` + + ${positional} + ${action.arguments._.description}` + } + + let usage = ` ${action_name}${ (options != "") ? " [options...]" : "" }${positional} + ${action.description}${positional_desc} +${options}` + + console.log(usage) +} + + let printpluginUsage = (plugin) => { let usage = ` CLI plugin: @@ -64,6 +102,10 @@ Actions and Action Options: ` console.log(usage) + + for (const action in plugin.actions) { + printPluginActionUsage(plugin.actions[action], action) + } } //console.log(parsed_args) @@ -156,20 +198,25 @@ if (parsed_args._.length == 1) { let action = parsed_args._[1] if ( ! (action in plugin.actions) ) { - var exit_code = 4 + var exit_code = 0 if (!['--help', '-h'].includes(action)) { console.log(`\n*** Action not supported: ${action} ***`) - exit_code = 0 + exit_code = 4 } printpluginUsage(plugin) Deno.exit(exit_code) } +if (['--help', '-h'].includes(parsed_args._[2])) { + printpluginUsage(plugin) + Deno.exit(0) +} + var parsed_plugin_args = parsePluginActionArgs( // removing the plugin name and the method name parsed_args._.slice(2), plugin.actions[action].arguments - ) + ) //console.log(parsed_plugin_args) From 035d5d8c4bad4c40df862a355c07769521367867 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27rysiek=27=20Wo=C5=BAniak?= Date: Sat, 10 Dec 2022 18:40:30 +0000 Subject: [PATCH 11/38] basic-integrity: improvements of CLI plugin actions metadata --- plugins/basic-integrity/cli.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/plugins/basic-integrity/cli.js b/plugins/basic-integrity/cli.js index b59e6b8..c2f93ad 100644 --- a/plugins/basic-integrity/cli.js +++ b/plugins/basic-integrity/cli.js @@ -150,10 +150,11 @@ const pluginActions = { description: "calculate subresource integrity hashes for provided files", arguments: { _: { - description: "paths to individual pieces of content" + name: "file", + description: "paths of files to be processed" }, algorithm: { - description: "array of SubtleCrypto.digest-compatible algorithm names to use to calculate digests (default: \"SHA-256\")", + description: "SubtleCrypto.digest-compatible algorithm names to use to calculate digests (default: \"SHA-256\")", collect: true, string: true, default: "SHA-256" From 3f95e786bc769c9d94a3039c32150b62f7e2eb0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27rysiek=27=20Wo=C5=BAniak?= Date: Sat, 10 Dec 2022 22:01:28 +0000 Subject: [PATCH 12/38] =?UTF-8?q?cli:=20bugfix=20=E2=80=94=20gracefully=20?= =?UTF-8?q?handling=20lack=20of=20"arguments"=20key=20in=20action=20object?= =?UTF-8?q?=20(ref.=20#66)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cli/lrcli.js | 49 +++++++++++++++++++++++++++---------------------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/cli/lrcli.js b/cli/lrcli.js index 665d345..761c95c 100755 --- a/cli/lrcli.js +++ b/cli/lrcli.js @@ -46,31 +46,35 @@ Options: let printPluginActionUsage = (action, action_name) => { + // initialize let options = "" - for (const opt in action.arguments) { - if (opt == '_') { - continue - } - options += `\n --${opt}` - if ("default" in action.arguments[opt]) { - options += ` (default: ${action.arguments[opt].default})` - } - options += `\n ${action.arguments[opt].description}` - options += `\n` - } + let positional = "" + let positional_desc = "" - let positional = '' - let positional_desc = '' - if ('_' in action.arguments) { - if ('name' in action.arguments._) { - positional = ` <${action.arguments._.name}...>` - } else { - positional = ' ' + if ("arguments" in action) { + for (const opt in action.arguments) { + if (opt == '_') { + continue + } + options += `\n --${opt}` + if ("default" in action.arguments[opt]) { + options += ` (default: ${action.arguments[opt].default})` + } + options += `\n ${action.arguments[opt].description}` + options += `\n` } - positional_desc = ` + + if ('_' in action.arguments) { + if ('name' in action.arguments._) { + positional = ` <${action.arguments._.name}...>` + } else { + positional = ' ' + } + positional_desc = ` - ${positional} - ${action.arguments._.description}` + ${positional} + ${action.arguments._.description}` + } } let usage = ` ${action_name}${ (options != "") ? " [options...]" : "" }${positional} @@ -215,7 +219,8 @@ if (['--help', '-h'].includes(parsed_args._[2])) { var parsed_plugin_args = parsePluginActionArgs( // removing the plugin name and the method name parsed_args._.slice(2), - plugin.actions[action].arguments + // empty object in case arguments key does not exist + plugin.actions[action].arguments || {} ) //console.log(parsed_plugin_args) From 8406d4bc5961a4df7f39f90b9edef65451cc9891 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27rysiek=27=20Wo=C5=BAniak?= Date: Sat, 10 Dec 2022 22:02:18 +0000 Subject: [PATCH 13/38] cli: signed-integrity cli implementation started (ref. #66) --- plugins/signed-integrity/cli.js | 49 +++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 plugins/signed-integrity/cli.js diff --git a/plugins/signed-integrity/cli.js b/plugins/signed-integrity/cli.js new file mode 100644 index 0000000..fee34ec --- /dev/null +++ b/plugins/signed-integrity/cli.js @@ -0,0 +1,49 @@ +/* ========================================================================= *\ +|* === Signed Integrity: content integrity using signed integrity data === *| +\* ========================================================================= */ + +/** + * signed-integrity plugin's deploy/utility functions + * + * this code expects a Deno runtime: + * https://deno.land/ + */ + + +/** + * generate an ECDSA P-384 keypair and export it as a JWK + * https://developer.mozilla.org/en-US/docs/Web/API/SubtleCrypto/importKey#json_web_key + */ +let genKeypair = async () => { + let keypair = await crypto.subtle.generateKey({ + name: "ECDSA", + namedCurve: "P-384" + }, + true, + ["sign", "verify"] + ); + let exported_keypair = { + publicKey: await crypto.subtle.exportKey("jwk", keypair.publicKey), + privateKey: await crypto.subtle.exportKey("jwk", keypair.privateKey) + } + return JSON.stringify(exported_keypair) +} + + +// this never changes +const pluginName = "signed-integrity" +const pluginDescription = "Fetching signed integrity data and using it to verify content.\nCLI used to generate subresource integrity tokens and save them in integrity files." +const pluginVersion = 'COMMIT_UNKNOWN' +const pluginActions = { + "gen-keypair": { + run: genKeypair, + description: "generate a keypair and export it as a JSON Web Key" + } +} + +export { + pluginName as name, + pluginDescription as description, + pluginVersion as version, + pluginActions as actions +} From 81fa42176b9d3e223f113263aff0ccc89343379f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27rysiek=27=20Wo=C5=BAniak?= Date: Sat, 10 Dec 2022 23:07:21 +0000 Subject: [PATCH 14/38] cli: lrcli.js minor formatting improvement (ref. #66) --- cli/lrcli.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/lrcli.js b/cli/lrcli.js index 761c95c..85dbfa3 100755 --- a/cli/lrcli.js +++ b/cli/lrcli.js @@ -72,8 +72,8 @@ let printPluginActionUsage = (action, action_name) => { } positional_desc = ` - ${positional} - ${action.arguments._.description}` + ${positional} + ${action.arguments._.description}` } } From 4c0d9f11672364c786941b9fcbac5dfc9cef2dd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27rysiek=27=20Wo=C5=BAniak?= Date: Sat, 10 Dec 2022 23:07:55 +0000 Subject: [PATCH 15/38] cli: signed-integrity get-pubkey action implemented (ref. #66) --- plugins/signed-integrity/cli.js | 40 +++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/plugins/signed-integrity/cli.js b/plugins/signed-integrity/cli.js index fee34ec..50abf6f 100644 --- a/plugins/signed-integrity/cli.js +++ b/plugins/signed-integrity/cli.js @@ -29,6 +29,36 @@ let genKeypair = async () => { return JSON.stringify(exported_keypair) } +/** + * derive a public key from a provided public key file + * + * keyfile - a path to a file containing the private key + */ +let getPubkey = async (keyfile) => { + + // we only want to process one file + if (Array.isArray(keyfile)) { + keyfile = keyfile[0] + } + // + var keydata = JSON.parse(await Deno.readTextFile(keyfile)); + // the key can be eitehr in a CryptoKeyPair structure, or directly in CryptoKey structure + // standardize! + if ("privateKey" in keydata) { + keydata = keydata.privateKey + } + + // make the key public by deleting private parts and modifying key_ops + // ref. https://stackoverflow.com/a/57571350 + delete keydata.d; + keydata.key_ops = ['verify'] + + // import the key, thus making sure data is valid and makes sense + let key = await crypto.subtle.importKey("jwk", keydata, {name: 'ECDSA', namedCurve: 'P-384'}, true, ['verify']) + + // export it again + return JSON.stringify(await crypto.subtle.exportKey("jwk", key)) +} // this never changes const pluginName = "signed-integrity" @@ -38,6 +68,16 @@ const pluginActions = { "gen-keypair": { run: genKeypair, description: "generate a keypair and export it as a JSON Web Key" + }, + "get-pubkey": { + run: getPubkey, + description: "print out a public key derived from the provided private key", + arguments: { + '_': { + name: "keyfile", + description: "file containing the private key in JSON Web Key format" + } + } } } From 5af7012da798badee22145dc32a008ad324eb89a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27rysiek=27=20Wo=C5=BAniak?= Date: Sat, 10 Dec 2022 23:48:14 +0000 Subject: [PATCH 16/38] cli: basic-integrity function comment added (ref. #66) --- plugins/basic-integrity/cli.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/plugins/basic-integrity/cli.js b/plugins/basic-integrity/cli.js index c2f93ad..2bc29b1 100644 --- a/plugins/basic-integrity/cli.js +++ b/plugins/basic-integrity/cli.js @@ -26,6 +26,12 @@ let binToBase64 = (binary_data) => { } +/** + * get integrity digests for a given path + * + * path - path to a file whose digest is to be generated + * algos - array of SubtleCrypto.digest-compatible algorithm names + */ let getFileIntegrity = async (path, algos) => { var result = [] From c0f6f62cfeccb45c26a60c69ec2c8ee1db65fe50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27rysiek=27=20Wo=C5=BAniak?= Date: Sat, 10 Dec 2022 23:49:10 +0000 Subject: [PATCH 17/38] cli: signed-integrity - started implementing the gen-integrity action (ref. #66) --- plugins/signed-integrity/cli.js | 160 +++++++++++++++++++++++++++++++- 1 file changed, 157 insertions(+), 3 deletions(-) diff --git a/plugins/signed-integrity/cli.js b/plugins/signed-integrity/cli.js index 50abf6f..6f405f8 100644 --- a/plugins/signed-integrity/cli.js +++ b/plugins/signed-integrity/cli.js @@ -10,6 +10,22 @@ */ +/** + * helper function, converting binary to base64 + * this need not be extremely fast, since it will only be used on digests + * + * binary_data - data to convert to base64 + */ +let binToBase64 = (binary_data) => { + return btoa( + (new Uint8Array(binary_data)) + .reduce((bin, byte)=>{ + return bin += String.fromCharCode(byte) + }, '') + ) +} + + /** * generate an ECDSA P-384 keypair and export it as a JWK * https://developer.mozilla.org/en-US/docs/Web/API/SubtleCrypto/importKey#json_web_key @@ -30,7 +46,7 @@ let genKeypair = async () => { } /** - * derive a public key from a provided public key file + * get a public key from a provided private key file * * keyfile - a path to a file containing the private key */ @@ -42,7 +58,7 @@ let getPubkey = async (keyfile) => { } // var keydata = JSON.parse(await Deno.readTextFile(keyfile)); - // the key can be eitehr in a CryptoKeyPair structure, or directly in CryptoKey structure + // the key can be either in a CryptoKeyPair structure, or directly in CryptoKey structure // standardize! if ("privateKey" in keydata) { keydata = keydata.privateKey @@ -54,12 +70,131 @@ let getPubkey = async (keyfile) => { keydata.key_ops = ['verify'] // import the key, thus making sure data is valid and makes sense - let key = await crypto.subtle.importKey("jwk", keydata, {name: 'ECDSA', namedCurve: 'P-384'}, true, ['verify']) + let key = await crypto.subtle.importKey( + "jwk", + keydata, + { + name: 'ECDSA', + namedCurve: 'P-384' + }, + true, + ['verify'] + ) // export it again return JSON.stringify(await crypto.subtle.exportKey("jwk", key)) } +/** + * get integrity digests for a given path + * + * path - path to a file whose digest is to be generated + * algos - array of SubtleCrypto.digest-compatible algorithm names + */ +let getFileIntegrity = async (path, algos) => { + + var result = [] + + // open the file and get some info on it + const file = await Deno.open( + path, + { read: true } + ); + const fileInfo = await file.stat(); + + // are we working with a file? + if (fileInfo.isFile) { + + //console.log(`+-- reading: ${path}`) + + // initialize + var content = new Uint8Array() + var buf = new Uint8Array(1000); + + // read the first batch + var numread = file.readSync(buf); + + // read the rest, if there is anything to read + while (numread !== null) { + //console.log(` +-- read: ${numread}`) + //console.log(` +-- length: ${content.length}`) + + // there has to be a better way... + var new_content = new Uint8Array(content.length + numread); + //console.log(` +-- new length: ${new_content.length}`) + new_content.set(content) + if (buf.length === numread) { + new_content.set(buf, content.length) + } else { + new_content.set(buf.slice(0, numread), content.length) + } + content = new_content + + // read some more + numread = file.readSync(buf); + } + //console.log(' +-- done.') + + //console.log('+-- calculating digests') + for (const algo of algos) { + //console.log(` +-- ${algo}`) + var digest = algo.toLowerCase().replace('-', '') + '-' + binToBase64(await crypto.subtle.digest(algo, content)) + //console.log(digest) + result.push(digest) + } + //console.log(`+-- file done: ${path}`) + + // we are not working with a file + } else { + result = false; + } + + // putting this in a try-catch block as the file + // is apparently being auto-closed? + // https://issueantenna.com/repo/denoland/deno/issues/15442 + try { + await file.close(); + } catch (BadResource) {} + + // return the result + return result +} + + +/** + * generate integrity files for provided paths + * + * paths - paths to files for which integrity files are to be generated + * keyfile - path of the file containing the private key to use + * extension - file extension to use when saving integrity files (default: ".integrity") + */ +let genSignedIntegrity = async (paths, keyfile, extension='.integrity') => { + + // load the key + var keydata = JSON.parse(await Deno.readTextFile(keyfile)); + + // the key can be either in a CryptoKeyPair structure, or directly in CryptoKey structure + // standardize! + if ("privateKey" in keydata) { + keydata = keydata.privateKey + } + + // import the key, thus making sure data is valid and makes sense + let privkey = await crypto.subtle.importKey( + "jwk", + keydata, + { + name: 'ECDSA', + namedCurve: 'P-384' + + }, + true, + ['sign'] + ) + + // TODO: implement shit +} + // this never changes const pluginName = "signed-integrity" const pluginDescription = "Fetching signed integrity data and using it to verify content.\nCLI used to generate subresource integrity tokens and save them in integrity files." @@ -78,6 +213,25 @@ const pluginActions = { description: "file containing the private key in JSON Web Key format" } } + }, + "gen-integrity": { + run: genSignedIntegrity, + description: "generate integrity files for given paths", + arguments: { + '_': { + name: "file", + description: "paths to generate signed integrity files for" + }, + keyfile: { + description: "path to the file containing a private key in JSON Web Key format", + string: true + }, + extension: { + description: "file extension to use when saving integrity files", + default: '.integrity', + string: true + } + } } } From 020df117ab20b6bc3f0af5f8c845cb6b4fe06f6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27rysiek=27=20Wo=C5=BAniak?= Date: Sun, 11 Dec 2022 02:42:55 +0000 Subject: [PATCH 18/38] cli: comment todo in lrcli added (ref. #66) --- cli/lrcli.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cli/lrcli.js b/cli/lrcli.js index 85dbfa3..ffd5a22 100755 --- a/cli/lrcli.js +++ b/cli/lrcli.js @@ -1,4 +1,6 @@ -#!/usr/bin/env -S deno run --allow-read +#!/usr/bin/env -S deno run --allow-read --allow-write + +// TODO: handle the permissions better... somehow? import { parse } from "https://deno.land/std/flags/mod.ts"; From 5c6a8d8d6c668fa9473d6ed38f0194b9500f8c62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27rysiek=27=20Wo=C5=BAniak?= Date: Sun, 11 Dec 2022 03:06:47 +0000 Subject: [PATCH 19/38] cli: signed-integrity CLI can now generate... signed integrity data (ref. #66) --- plugins/signed-integrity/cli.js | 62 ++++++++++++++++++++++++++++++--- 1 file changed, 57 insertions(+), 5 deletions(-) diff --git a/plugins/signed-integrity/cli.js b/plugins/signed-integrity/cli.js index 6f405f8..92311f0 100644 --- a/plugins/signed-integrity/cli.js +++ b/plugins/signed-integrity/cli.js @@ -56,8 +56,8 @@ let getPubkey = async (keyfile) => { if (Array.isArray(keyfile)) { keyfile = keyfile[0] } - // - var keydata = JSON.parse(await Deno.readTextFile(keyfile)); + // get the key data from the key file + var keydata = JSON.parse(Deno.readTextFileSync(keyfile)); // the key can be either in a CryptoKeyPair structure, or directly in CryptoKey structure // standardize! if ("privateKey" in keydata) { @@ -166,12 +166,13 @@ let getFileIntegrity = async (path, algos) => { * * paths - paths to files for which integrity files are to be generated * keyfile - path of the file containing the private key to use + * output - whether to output the signed integrity data to "files" or "stdout" (default) * extension - file extension to use when saving integrity files (default: ".integrity") */ -let genSignedIntegrity = async (paths, keyfile, extension='.integrity') => { +let genSignedIntegrity = async (paths, keyfile, output='files', extension='.integrity') => { // load the key - var keydata = JSON.parse(await Deno.readTextFile(keyfile)); + var keydata = JSON.parse(Deno.readTextFileSync(keyfile)); // the key can be either in a CryptoKeyPair structure, or directly in CryptoKey structure // standardize! @@ -192,7 +193,53 @@ let genSignedIntegrity = async (paths, keyfile, extension='.integrity') => { ['sign'] ) - // TODO: implement shit + // initialize the result + let result = '' + + // do the thing for each path + for (const path of paths) { + + // get the integrity hash + let integrity = await getFileIntegrity(path, ["SHA-512"]) + + // if integrity is false, the path is a directory or some such + if (integrity == false) { + continue; + } + + // JWT header + let header = btoa('{"alg": "ES384"}').replace(/\//g, '_').replace(/\+/g, '-').replace(/=/g, '') + + // JWT payload -- the integrity hash + let payload = btoa(`{"integrity": "${integrity[0]}"}`).replace(/\//g, '_').replace(/\+/g, '-').replace(/=/g, '') + + // get the signature for header + payload + let data = new TextEncoder("utf-8").encode(header + '.' + payload) + let signature = new Uint8Array(await crypto.subtle.sign( + { + name: "ECDSA", + hash: {name: "SHA-384"} + }, + privkey, + data + )) + // and prepare it for inclusion in the JWT + signature = binToBase64(signature).replace(/\//g, '_').replace(/\+/g, '-').replace(/=/g, '') + + // put it all together + let jwt = header + '.' + payload + '.' + signature + + // do we want output to stdout or files + if (output == 'stdout') { + result += `${path}: ${jwt}\n` + } else { + // write it out to {path}.extension + Deno.writeTextFileSync(path + extension, jwt) + } + } + + // return whatever we have to return + return result; } // this never changes @@ -226,6 +273,11 @@ const pluginActions = { description: "path to the file containing a private key in JSON Web Key format", string: true }, + output: { + description: "output mode: 'files' or 'stdout'", + default: 'stdout', + string: true + }, extension: { description: "file extension to use when saving integrity files", default: '.integrity', From 6e4e29b47e93de0f626458b0ed801aba283e7e5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27rysiek=27=20Wo=C5=BAniak?= Date: Sun, 11 Dec 2022 14:15:02 +0000 Subject: [PATCH 20/38] cli: basic-integrity minor action description fix (ref. #66) --- plugins/basic-integrity/cli.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/basic-integrity/cli.js b/plugins/basic-integrity/cli.js index 2bc29b1..44a122d 100644 --- a/plugins/basic-integrity/cli.js +++ b/plugins/basic-integrity/cli.js @@ -160,7 +160,7 @@ const pluginActions = { description: "paths of files to be processed" }, algorithm: { - description: "SubtleCrypto.digest-compatible algorithm names to use to calculate digests (default: \"SHA-256\")", + description: "SubtleCrypto.digest-compatible algorithm names to use when calculating digests (default: \"SHA-256\")", collect: true, string: true, default: "SHA-256" From 4d80cbe7436e885fec20cddd4a4599335004d8e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27rysiek=27=20Wo=C5=BAniak?= Date: Sun, 11 Dec 2022 14:15:50 +0000 Subject: [PATCH 21/38] cli: signed-integrity now takes "algorithm" option (ref. #66) --- plugins/signed-integrity/cli.js | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/plugins/signed-integrity/cli.js b/plugins/signed-integrity/cli.js index 92311f0..5852303 100644 --- a/plugins/signed-integrity/cli.js +++ b/plugins/signed-integrity/cli.js @@ -166,10 +166,16 @@ let getFileIntegrity = async (path, algos) => { * * paths - paths to files for which integrity files are to be generated * keyfile - path of the file containing the private key to use - * output - whether to output the signed integrity data to "files" or "stdout" (default) + * algos - array of SubtleCrypto.digest-compatible hashing algorithms (default: ["SHA-256"]) + * output - whether to output the signed integrity data to "files" or "text" (default) * extension - file extension to use when saving integrity files (default: ".integrity") */ -let genSignedIntegrity = async (paths, keyfile, output='files', extension='.integrity') => { +let genSignedIntegrity = async ( + paths, + keyfile, + algos=["SHA-256"], + output='files', + extension='.integrity') => { // load the key var keydata = JSON.parse(Deno.readTextFileSync(keyfile)); @@ -200,7 +206,7 @@ let genSignedIntegrity = async (paths, keyfile, output='files', extension='.inte for (const path of paths) { // get the integrity hash - let integrity = await getFileIntegrity(path, ["SHA-512"]) + let integrity = await getFileIntegrity(path, algos) // if integrity is false, the path is a directory or some such if (integrity == false) { @@ -211,7 +217,10 @@ let genSignedIntegrity = async (paths, keyfile, output='files', extension='.inte let header = btoa('{"alg": "ES384"}').replace(/\//g, '_').replace(/\+/g, '-').replace(/=/g, '') // JWT payload -- the integrity hash - let payload = btoa(`{"integrity": "${integrity[0]}"}`).replace(/\//g, '_').replace(/\+/g, '-').replace(/=/g, '') + // from MDN: "An integrity value may contain multiple hashes separated by whitespace. + // A resource will be loaded if it matches one of those hashes." + // https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity + let payload = btoa(`{"integrity": "${integrity.join(' ')}"}`).replace(/\//g, '_').replace(/\+/g, '-').replace(/=/g, '') // get the signature for header + payload let data = new TextEncoder("utf-8").encode(header + '.' + payload) @@ -229,8 +238,8 @@ let genSignedIntegrity = async (paths, keyfile, output='files', extension='.inte // put it all together let jwt = header + '.' + payload + '.' + signature - // do we want output to stdout or files - if (output == 'stdout') { + // do we want output to text or files + if (output == 'text') { result += `${path}: ${jwt}\n` } else { // write it out to {path}.extension @@ -273,9 +282,15 @@ const pluginActions = { description: "path to the file containing a private key in JSON Web Key format", string: true }, + algorithm: { + description: "SubtleCrypto.digest-compatible algorithm names to use when calculating digests (default: \"SHA-256\")", + collect: true, + string: true, + default: "SHA-256" + }, output: { - description: "output mode: 'files' or 'stdout'", - default: 'stdout', + description: "output mode: 'files' or 'text'", + default: 'text', string: true }, extension: { From b226bf301275cd6e09401ee8041fb7fbcdd05583 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27rysiek=27=20Wo=C5=BAniak?= Date: Sun, 11 Dec 2022 14:38:57 +0000 Subject: [PATCH 22/38] cli: signed-integrity now has "json" output mode, and it is default (ref. #66) --- plugins/signed-integrity/cli.js | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/plugins/signed-integrity/cli.js b/plugins/signed-integrity/cli.js index 5852303..313170d 100644 --- a/plugins/signed-integrity/cli.js +++ b/plugins/signed-integrity/cli.js @@ -167,14 +167,14 @@ let getFileIntegrity = async (path, algos) => { * paths - paths to files for which integrity files are to be generated * keyfile - path of the file containing the private key to use * algos - array of SubtleCrypto.digest-compatible hashing algorithms (default: ["SHA-256"]) - * output - whether to output the signed integrity data to "files" or "text" (default) + * output - whether to output the signed integrity data to "files", or as "text" or "json" (default) * extension - file extension to use when saving integrity files (default: ".integrity") */ let genSignedIntegrity = async ( paths, keyfile, algos=["SHA-256"], - output='files', + output='json', extension='.integrity') => { // load the key @@ -200,7 +200,7 @@ let genSignedIntegrity = async ( ) // initialize the result - let result = '' + let result = {} // do the thing for each path for (const path of paths) { @@ -239,16 +239,21 @@ let genSignedIntegrity = async ( let jwt = header + '.' + payload + '.' + signature // do we want output to text or files - if (output == 'text') { - result += `${path}: ${jwt}\n` - } else { + result[path] = jwt + if (output == 'files') { // write it out to {path}.extension Deno.writeTextFileSync(path + extension, jwt) } } // return whatever we have to return - return result; + if (output == 'json') { + return JSON.stringify(result); + } else if (output == 'text') { + return Object.keys(result).map(p=>`${p}: ${result[p]}`).join('\n') + '\n' + } else if (output == 'files') { + return "created integrity files:\n" + Object.keys(result).map(p=>`- ${p}${extension}`).join('\n') + "\n" + } } // this never changes @@ -289,8 +294,8 @@ const pluginActions = { default: "SHA-256" }, output: { - description: "output mode: 'files' or 'text'", - default: 'text', + description: "output mode: 'files', 'text', or 'json'", + default: 'json', string: true }, extension: { From 10b42c52731aa0f4722d10ffb7d72131771682c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27rysiek=27=20Wo=C5=BAniak?= Date: Sun, 11 Dec 2022 15:00:14 +0000 Subject: [PATCH 23/38] cli: signed-integrity error checking in action handlers (ref. #66) --- plugins/signed-integrity/cli.js | 39 ++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/plugins/signed-integrity/cli.js b/plugins/signed-integrity/cli.js index 313170d..e6c2328 100644 --- a/plugins/signed-integrity/cli.js +++ b/plugins/signed-integrity/cli.js @@ -56,8 +56,20 @@ let getPubkey = async (keyfile) => { if (Array.isArray(keyfile)) { keyfile = keyfile[0] } - // get the key data from the key file - var keydata = JSON.parse(Deno.readTextFileSync(keyfile)); + + // we need non-empty arguments + if ((typeof keyfile !== "string") || (keyfile == "")) { + throw new Error("No keyfile provided.") + } + + // load the key + try { + var keydata = JSON.parse(Deno.readTextFileSync(keyfile)); + } catch(e) { + throw new Error(`Failed to load private key from '${keyfile}': ${e.message}`, {cause: e}) + } + + // the key can be either in a CryptoKeyPair structure, or directly in CryptoKey structure // standardize! if ("privateKey" in keydata) { @@ -177,8 +189,29 @@ let genSignedIntegrity = async ( output='json', extension='.integrity') => { + // we need non-empty arguments + if (!Array.isArray(paths) || (paths.length == 0)) { + throw new Error("Expected non-empty list of paths to process.") + } + if ((typeof keyfile !== "string") || (keyfile == "")) { + throw new Error("No keyfile provided.") + } + if (!Array.isArray(algos) || (algos.length == 0)) { + throw new Error("Expected non-empty list of algorithms to use.") + } + if (!['json', 'text', 'files'].includes(output)) { + throw new Error("Expected 'json', 'text', or 'files' as output type.") + } + if ( (output == 'files') && ( (typeof extension !== "string") || (extension == "") ) ) { + throw new Error("No extension provided.") + } + // load the key - var keydata = JSON.parse(Deno.readTextFileSync(keyfile)); + try { + var keydata = JSON.parse(Deno.readTextFileSync(keyfile)); + } catch(e) { + throw new Error(`Failed to load private key from '${keyfile}': ${e.message}`, {cause: e}) + } // the key can be either in a CryptoKeyPair structure, or directly in CryptoKey structure // standardize! From 1711334e74acd0259fa52f588d3a4cbb1b5c54b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27rysiek=27=20Wo=C5=BAniak?= Date: Sun, 11 Dec 2022 19:49:22 +0000 Subject: [PATCH 24/38] cli: lrcli prepared for testing (ref. #66) --- cli/lrcli.js | 187 ++++++++++++++++++++++++++------------------------- 1 file changed, 95 insertions(+), 92 deletions(-) diff --git a/cli/lrcli.js b/cli/lrcli.js index ffd5a22..b1e882e 100755 --- a/cli/lrcli.js +++ b/cli/lrcli.js @@ -4,28 +4,7 @@ import { parse } from "https://deno.land/std/flags/mod.ts"; -var parsed_args = parse( - Deno.args, - { - default: { - h: false, - }, - stopEarly: true, - boolean: [ "h" ], - string: [], - alias: { - h: [ "help" ] - }, - collect: [], - negatable: [], - // a function which is invoked with a command line parameter not defined - // in the options configuration object. If the function returns false, - // the unknown option is not added to parsedArgs. - unknown: null - } -); - -let printUsage = () => { +let getUsage = () => { let usage = ` Command-line interface for LibResilient. @@ -42,11 +21,11 @@ Options: ` - console.log(usage) + return usage } -let printPluginActionUsage = (action, action_name) => { +let getPluginActionUsage = (action, action_name) => { // initialize let options = "" @@ -83,11 +62,11 @@ let printPluginActionUsage = (action, action_name) => { ${action.description}${positional_desc} ${options}` - console.log(usage) + return usage } -let printpluginUsage = (plugin) => { +let getPluginUsage = (plugin) => { let usage = ` CLI plugin: ${plugin.name} @@ -105,17 +84,15 @@ General Options: If plugin-name is provided, print usage information of that plugin. Actions and Action Options: + ` - - console.log(usage) - for (const action in plugin.actions) { - printPluginActionUsage(plugin.actions[action], action) + usage += getPluginActionUsage(plugin.actions[action], action) + '\n' } + + return usage } -//console.log(parsed_args) - let parsePluginActionArgs = (args, argdef) => { var plugin_args_config = { @@ -150,9 +127,7 @@ let parsePluginActionArgs = (args, argdef) => { } } - //console.log(plugin_args_config) var parsed = parse(args, plugin_args_config) - //console.log(parsed) var result = [] @@ -176,70 +151,98 @@ let parsePluginActionArgs = (args, argdef) => { // we *always* pass arguments to plugins as arrays of strings, // even if we only got one value -// no unknown parsed args? that means we have no plugin specified -if (parsed_args._.length == 0) { - printUsage() - Deno.exit(1) -} - -// try loading the plugin -let plugin -try { - plugin = await import(`../plugins/${parsed_args._[0]}/cli.js`); -} catch (e) { - // unable to load the plugin? bail with info - console.log(`\n*** ${e} ***`) - printUsage() - Deno.exit(2) -} - -// if we only had exactly one unknown arg, we only have the plugin name -// but no info from the user what to do with it -// → print plugin usage and exit -if (parsed_args._.length == 1) { - console.log('\n*** No action specified for plugin ***') - printpluginUsage(plugin) - Deno.exit(3) -} - -let action = parsed_args._[1] -if ( ! (action in plugin.actions) ) { - var exit_code = 0 - if (!['--help', '-h'].includes(action)) { - console.log(`\n*** Action not supported: ${action} ***`) - exit_code = 4 +let main = async () => { + + var parsed_args = parse( + Deno.args, + { + default: { + h: false, + }, + stopEarly: true, + boolean: [ "h" ], + string: [], + alias: { + h: [ "help" ] + }, + collect: [], + negatable: [], + // a function which is invoked with a command line parameter not defined + // in the options configuration object. If the function returns false, + // the unknown option is not added to parsedArgs. + unknown: null + } + ); + + // no unknown parsed args? that means we have no plugin specified + if (parsed_args._.length == 0) { + console.log(getUsage()) + return 1 } - printpluginUsage(plugin) - Deno.exit(exit_code) -} -if (['--help', '-h'].includes(parsed_args._[2])) { - printpluginUsage(plugin) - Deno.exit(0) -} + // try loading the plugin + let plugin + try { + plugin = await import(`../plugins/${parsed_args._[0]}/cli.js`); + } catch (e) { + // unable to load the plugin? bail with info + console.log(`\n*** ${e} ***`) + console.log(getUsage()) + return 2 + } -var parsed_plugin_args = parsePluginActionArgs( - // removing the plugin name and the method name - parsed_args._.slice(2), - // empty object in case arguments key does not exist - plugin.actions[action].arguments || {} - ) + // if we only had exactly one unknown arg, we only have the plugin name + // but no info from the user what to do with it + // → print plugin usage and exit + if (parsed_args._.length == 1) { + console.log('\n*** No action specified for plugin ***') + console.log(getPluginUsage(plugin)) + return 3 + } + + let action = parsed_args._[1] + if ( ! (action in plugin.actions) ) { + var exit_code = 0 + if (!['--help', '-h'].includes(action)) { + console.log(`\n*** Action not supported: ${action} ***`) + exit_code = 4 + } + console.log(getPluginUsage(plugin)) + return exit_code + } -//console.log(parsed_plugin_args) + if (['--help', '-h'].includes(parsed_args._[2])) { + console.log(getPluginUsage(plugin)) + return 0 + } -// not using console.log here because we want the *exact* output -// without any extra ending newlines -try { - await Deno.stdout.write( - new TextEncoder().encode( - await plugin.actions[action].run(...parsed_plugin_args) + var parsed_plugin_args = parsePluginActionArgs( + // removing the plugin name and the method name + parsed_args._.slice(2), + // empty object in case arguments key does not exist + plugin.actions[action].arguments || {} + ) + + // not using console.log here because we want the *exact* output + // without any extra ending newlines + try { + await Deno.stdout.write( + new TextEncoder().encode( + await plugin.actions[action].run(...parsed_plugin_args) + ) ) - ) -} catch (e) { - console.log(`\n*** ${e} ***`) - printpluginUsage(plugin) + } catch (e) { + console.log(`\n*** ${e} ***`) + console.log(getPluginUsage(plugin)) + } } +// export the main function +export { + main +} - -//console.log(new URL('', import.meta.url).toString().split('/').at(-1)) +// run only if we're the main module +if (import.meta.main) { + Deno.exit(await main()) +} From 4b46a588090373ac28618cf7a1f4a8c2ab0e229c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27rysiek=27=20Wo=C5=BAniak?= Date: Sun, 11 Dec 2022 21:48:12 +0000 Subject: [PATCH 25/38] cli: preparing lrcli.js for tests, part 2 (ref. #66) --- cli/lrcli.js | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/cli/lrcli.js b/cli/lrcli.js index b1e882e..0f46c48 100755 --- a/cli/lrcli.js +++ b/cli/lrcli.js @@ -151,10 +151,10 @@ let parsePluginActionArgs = (args, argdef) => { // we *always* pass arguments to plugins as arrays of strings, // even if we only got one value -let main = async () => { +let main = async (args) => { var parsed_args = parse( - Deno.args, + args, { default: { h: false, @@ -177,7 +177,11 @@ let main = async () => { // no unknown parsed args? that means we have no plugin specified if (parsed_args._.length == 0) { console.log(getUsage()) - return 1 + if (parsed_args.help) { + return 0; + } else { + return 1; + } } // try loading the plugin @@ -195,9 +199,15 @@ let main = async () => { // but no info from the user what to do with it // → print plugin usage and exit if (parsed_args._.length == 1) { - console.log('\n*** No action specified for plugin ***') + if (!parsed_args.help) { + console.log('\n*** No action specified for plugin ***') + } console.log(getPluginUsage(plugin)) - return 3 + if (parsed_args.help) { + return 0; + } else { + return 3; + } } let action = parsed_args._[1] @@ -231,9 +241,11 @@ let main = async () => { await plugin.actions[action].run(...parsed_plugin_args) ) ) + return 0 } catch (e) { console.log(`\n*** ${e} ***`) console.log(getPluginUsage(plugin)) + return 5 } } @@ -244,5 +256,5 @@ export { // run only if we're the main module if (import.meta.main) { - Deno.exit(await main()) + Deno.exit(await main(Deno.args)) } From 011c720b2dfc0765cddaf10159acfbe7ee0d4658 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27rysiek=27=20Wo=C5=BAniak?= Date: Sun, 11 Dec 2022 21:48:36 +0000 Subject: [PATCH 26/38] cli: first tests and mocks for lrcli (ref. #66) --- __denotests__/cli/lrcli.test.js | 64 ++++++++++++++++++++++++++++ __denotests__/importmap.json | 5 +++ __denotests__/mocks/simple-plugin.js | 20 +++++++++ 3 files changed, 89 insertions(+) create mode 100644 __denotests__/cli/lrcli.test.js create mode 100644 __denotests__/importmap.json create mode 100644 __denotests__/mocks/simple-plugin.js diff --git a/__denotests__/cli/lrcli.test.js b/__denotests__/cli/lrcli.test.js new file mode 100644 index 0000000..fb936c0 --- /dev/null +++ b/__denotests__/cli/lrcli.test.js @@ -0,0 +1,64 @@ +import { assertEquals } from "https://deno.land/std@0.167.0/testing/asserts.ts"; +import { main } from '../../cli/lrcli.js'; + +Deno.test("basic usage info", async () => { + // init + let r + // no args + r = await main([]) + assertEquals(r, 1) + // -h/--help + r = await main(['-h']) + assertEquals(r, 0) + r = await main(['--help']) + assertEquals(r, 0) +}); + +Deno.test("non-existent plugin handling", async () => { + // init + let r + // just the plugin name + r = await main(['no-such-plugin']) + assertEquals(r, 2) + // plugin name with different combinations of help flag + r = await main(['no-such-plugin', '--help']) + assertEquals(r, 2) + r = await main(['no-such-plugin', '-h']) + assertEquals(r, 2) + r = await main(['--help', 'no-such-plugin']) + assertEquals(r, 2) + r = await main(['-h', 'no-such-plugin']) + assertEquals(r, 2) +}); + +// +// tests below need an imports map +// + +Deno.test("plugin loading", async () => { + const r = await main(['simple-plugin', '--help']) + assertEquals(r, 0) +}); + +Deno.test("plugin action processing", async () => { + // init + let r + // non-existent action + r = await main(['simple-plugin', 'non-such-action']) + assertEquals(r, 4) + r = await main(['simple-plugin', '-h', 'non-such-action']) + assertEquals(r, 0) + r = await main(['simple-plugin', '--help', 'non-such-action']) + assertEquals(r, 0) + r = await main(['simple-plugin', 'non-such-action', '-h']) + assertEquals(r, 4) + r = await main(['simple-plugin', 'non-such-action', '--help']) + assertEquals(r, 4) + // action that exists + r = await main(['simple-plugin', 'test-action']) + assertEquals(r, 0) + r = await main(['simple-plugin', 'test-action', '-h']) + assertEquals(r, 0) + r = await main(['simple-plugin', 'test-action', '--help']) + assertEquals(r, 0) +}); diff --git a/__denotests__/importmap.json b/__denotests__/importmap.json new file mode 100644 index 0000000..fe3999a --- /dev/null +++ b/__denotests__/importmap.json @@ -0,0 +1,5 @@ +{ + "imports": { + "../plugins/simple-plugin/cli.js": "./mocks/simple-plugin.js" + } +} diff --git a/__denotests__/mocks/simple-plugin.js b/__denotests__/mocks/simple-plugin.js new file mode 100644 index 0000000..c0de6e8 --- /dev/null +++ b/__denotests__/mocks/simple-plugin.js @@ -0,0 +1,20 @@ +let testAction = () => { + return "testAction run!" +} + +const pluginName = "simple-plugin" +const pluginDescription = "A simple plugin for testing" +const pluginVersion = "0.0.1" +const pluginActions = { + 'test-action': { + run: testAction, + description: "Test action of a test plugin" + } +} + +export { + pluginName as name, + pluginDescription as description, + pluginVersion as version, + pluginActions as actions +} From 16a2122629466cdc94d01a7ee17daf59b802aa26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27rysiek=27=20Wo=C5=BAniak?= Date: Sun, 11 Dec 2022 21:57:39 +0000 Subject: [PATCH 27/38] cli: .gitlab-ci.yml updated for running deno tests (ref. #66) --- .gitlab-ci.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a0cb5f1..9c6c839 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -27,6 +27,15 @@ libresilient-test: tags: - docker - linux + +lrcli-test: + image: deno:1.28.3 + stage: test + script: + - deno test --quiet --allow-read --import-map=__denotests__/importmap.json __denotests__/ + tags: + - docker + - linux stages: - test From cb53fd16f413559b99e2f2572b0813f0b7b7cf3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27rysiek=27=20Wo=C5=BAniak?= Date: Sun, 11 Dec 2022 21:59:22 +0000 Subject: [PATCH 28/38] bugfix for gitlab-ci config (ref. #66) --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9c6c839..7de5830 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -29,7 +29,7 @@ libresilient-test: - linux lrcli-test: - image: deno:1.28.3 + image: denoland/deno:1.28.3 stage: test script: - deno test --quiet --allow-read --import-map=__denotests__/importmap.json __denotests__/ From c6e04e9ca00d2c96bcd74f198d28762f64c19105 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27rysiek=27=20Wo=C5=BAniak?= Date: Sun, 11 Dec 2022 22:17:29 +0000 Subject: [PATCH 29/38] jest config updated to ignore deno tests (ref #66) --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 3c764db..ece6b15 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "cobertura" ], "testMatch": [ - "**/*.test.js" + "**/__tests__/**/*.test.js" ] }, "devDependencies": { From 78b37cb7f05f955b9b69521be90b5f38fadacd5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27rysiek=27=20Wo=C5=BAniak?= Date: Mon, 12 Dec 2022 23:30:24 +0000 Subject: [PATCH 30/38] cli: initial deno tests for basic-integrity cli plugin (ref. #66) --- __denotests__/plugins/basic-integrity.test.js | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 __denotests__/plugins/basic-integrity.test.js diff --git a/__denotests__/plugins/basic-integrity.test.js b/__denotests__/plugins/basic-integrity.test.js new file mode 100644 index 0000000..ce81405 --- /dev/null +++ b/__denotests__/plugins/basic-integrity.test.js @@ -0,0 +1,46 @@ +import { assert } from "https://deno.land/std@0.167.0/testing/asserts.ts"; + +Deno.test("plugin load", async () => { + const bi = await import('../../plugins/basic-integrity/cli.js') + assert("name" in bi) + assert(bi.name == "basic-integrity") + assert("description" in bi) + assert("actions" in bi) +}); + +Deno.test("get-integrity action defined", async () => { + const bi = await import('../../plugins/basic-integrity/cli.js') + assert("get-integrity" in bi.actions) + const gi = bi.actions["get-integrity"] + assert("run" in gi) + assert("description" in gi) + assert("arguments" in gi) + const gia = gi.arguments + assert("_" in gia) + assert("algorithm" in gia) + assert("output" in gia) + assert("name" in gia._) + assert("description" in gia._) + assert("description" in gia.algorithm) + assert("collect" in gia.algorithm) + assert(gia.algorithm.collect) + assert("string" in gia.algorithm) + assert(gia.algorithm.string) + assert("description" in gia.output) + assert("collect" in gia.output) + assert(!gia.output.collect) + assert("string" in gia.output) + assert(gia.output.string) +}); + +// this is a separate test in order to catch any changing defaults +Deno.test("get-integrity action defaults", async () => { + const bi = await import('../../plugins/basic-integrity/cli.js') + const gia = bi.actions["get-integrity"].arguments + assert("default" in gia.algorithm) + assert(gia.algorithm.default == "SHA-256") + assert("default" in gia.output) + assert(gia.output.default == "json") +}); + +//TODO cont. From 1d8bcc0d3987a2b8d368eb7f516894bf31160e00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27rysiek=27=20Wo=C5=BAniak?= Date: Tue, 13 Dec 2022 23:43:42 +0000 Subject: [PATCH 31/38] cli: basic-integrity test suite complete (ref. #66) --- __denotests__/mocks/hello.txt | 1 + __denotests__/plugins/basic-integrity.test.js | 51 ++++++++++++++++++- 2 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 __denotests__/mocks/hello.txt diff --git a/__denotests__/mocks/hello.txt b/__denotests__/mocks/hello.txt new file mode 100644 index 0000000..95d09f2 --- /dev/null +++ b/__denotests__/mocks/hello.txt @@ -0,0 +1 @@ +hello world \ No newline at end of file diff --git a/__denotests__/plugins/basic-integrity.test.js b/__denotests__/plugins/basic-integrity.test.js index ce81405..1893f70 100644 --- a/__denotests__/plugins/basic-integrity.test.js +++ b/__denotests__/plugins/basic-integrity.test.js @@ -1,4 +1,9 @@ -import { assert } from "https://deno.land/std@0.167.0/testing/asserts.ts"; +import { + assert, + assertThrows, + assertRejects, + assertEquals +} from "https://deno.land/std@0.167.0/testing/asserts.ts"; Deno.test("plugin load", async () => { const bi = await import('../../plugins/basic-integrity/cli.js') @@ -43,4 +48,46 @@ Deno.test("get-integrity action defaults", async () => { assert(gia.output.default == "json") }); -//TODO cont. +Deno.test("get-integrity verifies arguments are sane", async () => { + const bi = await import('../../plugins/basic-integrity/cli.js') + const gi = bi.actions["get-integrity"] + assertRejects(gi.run, Error, "Expected non-empty list of files to generate digests of.") + assertRejects(async ()=>{ + await gi.run(['no-such-file']) + }, Error, "No such file or directory") + assertRejects(async ()=>{ + await gi.run(['irrelevant'], []) + }, Error, "Expected non-empty list of algorithms to use.") + assertRejects(async ()=>{ + await gi.run(['irrelevant'], ['SHA-384'], false) + }, Error, "Expected either 'json' or 'text' as output type to generate.") +}); + +Deno.test("get-integrity handles paths in a sane way", async () => { + const bi = await import('../../plugins/basic-integrity/cli.js') + const gi = bi.actions["get-integrity"] + assertEquals(await gi.run(['./']), '{}') + assertEquals(await gi.run(['./__denotests__/mocks/hello.txt']), '{"./__denotests__/mocks/hello.txt":["sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek="]}') + assertEquals(await gi.run(['./', './__denotests__/mocks/hello.txt']), '{"./__denotests__/mocks/hello.txt":["sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek="]}') +}); + +Deno.test("get-integrity handles algos argument in a sane way", async () => { + const bi = await import('../../plugins/basic-integrity/cli.js') + const gi = bi.actions["get-integrity"] + assertRejects(async ()=>{ + await gi.run(['./__denotests__/mocks/hello.txt'], ['BAD-ALG']) + }, Error, 'Unrecognized algorithm name') + assertEquals(await gi.run(['./__denotests__/mocks/hello.txt'], ['SHA-256']), '{"./__denotests__/mocks/hello.txt":["sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek="]}') + assertEquals(await gi.run(['./__denotests__/mocks/hello.txt'], ['SHA-384']), '{"./__denotests__/mocks/hello.txt":["sha384-/b2OdaZ/KfcBpOBAOF4uI5hjA+oQI5IRr5B/y7g1eLPkF8txzmRu/QgZ3YwIjeG9"]}') + assertEquals(await gi.run(['./__denotests__/mocks/hello.txt'], ['SHA-512']), '{"./__denotests__/mocks/hello.txt":["sha512-MJ7MSJwS1utMxA9QyQLytNDtd+5RGnx6m808qG1M2G+YndNbxf9JlnDaNCVbRbDP2DDoH2Bdz33FVC6TrpzXbw=="]}') + assertEquals(await gi.run(['./__denotests__/mocks/hello.txt'], ['SHA-256', 'SHA-384', 'SHA-512']), '{"./__denotests__/mocks/hello.txt":["sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek=","sha384-/b2OdaZ/KfcBpOBAOF4uI5hjA+oQI5IRr5B/y7g1eLPkF8txzmRu/QgZ3YwIjeG9","sha512-MJ7MSJwS1utMxA9QyQLytNDtd+5RGnx6m808qG1M2G+YndNbxf9JlnDaNCVbRbDP2DDoH2Bdz33FVC6TrpzXbw=="]}') +}); + +Deno.test("get-integrity handles output argument in a sane way", async () => { + const bi = await import('../../plugins/basic-integrity/cli.js') + const gi = bi.actions["get-integrity"] + assertEquals(await gi.run(['./__denotests__/mocks/hello.txt'], ['SHA-256'], 'text'), './__denotests__/mocks/hello.txt: sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek=\n') + assertEquals(await gi.run(['./__denotests__/mocks/hello.txt'], ['SHA-384'], 'text'), './__denotests__/mocks/hello.txt: sha384-/b2OdaZ/KfcBpOBAOF4uI5hjA+oQI5IRr5B/y7g1eLPkF8txzmRu/QgZ3YwIjeG9\n') + assertEquals(await gi.run(['./__denotests__/mocks/hello.txt'], ['SHA-512'], 'text'), './__denotests__/mocks/hello.txt: sha512-MJ7MSJwS1utMxA9QyQLytNDtd+5RGnx6m808qG1M2G+YndNbxf9JlnDaNCVbRbDP2DDoH2Bdz33FVC6TrpzXbw==\n') + assertEquals(await gi.run(['./__denotests__/mocks/hello.txt'], ['SHA-256', 'SHA-384', 'SHA-512'], 'text'), './__denotests__/mocks/hello.txt: sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek= sha384-/b2OdaZ/KfcBpOBAOF4uI5hjA+oQI5IRr5B/y7g1eLPkF8txzmRu/QgZ3YwIjeG9 sha512-MJ7MSJwS1utMxA9QyQLytNDtd+5RGnx6m808qG1M2G+YndNbxf9JlnDaNCVbRbDP2DDoH2Bdz33FVC6TrpzXbw==\n') +}); From 6dbffb34f9cbe72041f0e92217a1c775b5332a65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27rysiek=27=20Wo=C5=BAniak?= Date: Tue, 13 Dec 2022 23:44:12 +0000 Subject: [PATCH 32/38] cli: basic-integrity plugin bugfix (ref. #66) --- plugins/basic-integrity/cli.js | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/plugins/basic-integrity/cli.js b/plugins/basic-integrity/cli.js index 44a122d..83ad926 100644 --- a/plugins/basic-integrity/cli.js +++ b/plugins/basic-integrity/cli.js @@ -35,6 +35,7 @@ let binToBase64 = (binary_data) => { let getFileIntegrity = async (path, algos) => { var result = [] + var content = false // open the file and get some info on it const file = await Deno.open( @@ -49,7 +50,7 @@ let getFileIntegrity = async (path, algos) => { //console.log(`+-- reading: ${path}`) // initialize - var content = new Uint8Array() + content = new Uint8Array() var buf = new Uint8Array(1000); // read the first batch @@ -74,20 +75,6 @@ let getFileIntegrity = async (path, algos) => { // read some more numread = file.readSync(buf); } - //console.log(' +-- done.') - - //console.log('+-- calculating digests') - for (const algo of algos) { - //console.log(` +-- ${algo}`) - var digest = algo.toLowerCase().replace('-', '') + '-' + binToBase64(await crypto.subtle.digest(algo, content)) - //console.log(digest) - result.push(digest) - } - //console.log(`+-- file done: ${path}`) - - // we are not working with a file - } else { - result = false; } // putting this in a try-catch block as the file @@ -97,6 +84,20 @@ let getFileIntegrity = async (path, algos) => { await file.close(); } catch (BadResource) {} + // did we get any content? + if (typeof content != "boolean") { + for (const algo of algos) { + //console.log(` +-- ${algo}`) + var digest = algo.toLowerCase().replace('-', '') + '-' + binToBase64(await crypto.subtle.digest(algo, content)) + //console.log(digest) + result.push(digest) + } + // no content means not a file + } else { + // so no result + result = false + } + // return the result return result } From bae130d2a1253ed1fe3d6ab6083f205e237c22d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27rysiek=27=20Wo=C5=BAniak?= Date: Wed, 14 Dec 2022 23:05:05 +0000 Subject: [PATCH 33/38] cli: basic-integrity minor test description improvement (ref. #66) --- __denotests__/plugins/basic-integrity.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/__denotests__/plugins/basic-integrity.test.js b/__denotests__/plugins/basic-integrity.test.js index 1893f70..02d7abb 100644 --- a/__denotests__/plugins/basic-integrity.test.js +++ b/__denotests__/plugins/basic-integrity.test.js @@ -5,7 +5,7 @@ import { assertEquals } from "https://deno.land/std@0.167.0/testing/asserts.ts"; -Deno.test("plugin load", async () => { +Deno.test("plugin loads", async () => { const bi = await import('../../plugins/basic-integrity/cli.js') assert("name" in bi) assert(bi.name == "basic-integrity") From 8c0cda1adf5e53cd8c68cbdca93875cc43adaaf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27rysiek=27=20Wo=C5=BAniak?= Date: Wed, 14 Dec 2022 23:05:38 +0000 Subject: [PATCH 34/38] cli: denotests keyfile mock file (ref. #66) --- __denotests__/mocks/keyfile.json | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 __denotests__/mocks/keyfile.json diff --git a/__denotests__/mocks/keyfile.json b/__denotests__/mocks/keyfile.json new file mode 100644 index 0000000..8717502 --- /dev/null +++ b/__denotests__/mocks/keyfile.json @@ -0,0 +1,25 @@ +{ + "publicKey": { + "kty": "EC", + "crv": "P-384", + "alg": "ES384", + "x": "rrFawYTuFo8ZjoDxaztUU-c_RAwjw1Y9Tp3j4nH4WsY2Zlizf40Mvz_0BUkVVZCw", + "y": "HaFct6PVK2CQ7ZT2SHClnN-knmGfjY_DFwc6qrAu1s0DFZ8fEUuNdmkTlj9T4NQw", + "key_ops": [ + "verify" + ], + "ext": true + }, + "privateKey": { + "kty": "EC", + "crv": "P-384", + "alg": "ES384", + "x": "rrFawYTuFo8ZjoDxaztUU-c_RAwjw1Y9Tp3j4nH4WsY2Zlizf40Mvz_0BUkVVZCw", + "y": "HaFct6PVK2CQ7ZT2SHClnN-knmGfjY_DFwc6qrAu1s0DFZ8fEUuNdmkTlj9T4NQw", + "d": "zNpKLEqrWozLwaxlxnEa8njDLVQbwfHxX0OBNqB0jDl5qLLDcYF9NtHtXGzUWF5r", + "key_ops": [ + "sign" + ], + "ext": true + } +} From d6ac1698169a5fb57bd170832170d0bdfa8c470d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27rysiek=27=20Wo=C5=BAniak?= Date: Wed, 14 Dec 2022 23:06:01 +0000 Subject: [PATCH 35/38] cli: started implementing signed-integrity tests (ref. #66) --- .../plugins/signed-integrity.test.js | 135 ++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 __denotests__/plugins/signed-integrity.test.js diff --git a/__denotests__/plugins/signed-integrity.test.js b/__denotests__/plugins/signed-integrity.test.js new file mode 100644 index 0000000..fd74707 --- /dev/null +++ b/__denotests__/plugins/signed-integrity.test.js @@ -0,0 +1,135 @@ +import { + assert, + assertThrows, + assertRejects, + assertEquals +} from "https://deno.land/std@0.167.0/testing/asserts.ts"; + +Deno.test("plugin loads", async () => { + const bi = await import('../../plugins/signed-integrity/cli.js') + assert("name" in bi) + assert(bi.name == "signed-integrity") + assert("description" in bi) + assert("actions" in bi) +}); + +Deno.test("gen-integrity action defined", async () => { + const bi = await import('../../plugins/signed-integrity/cli.js') + assert("gen-integrity" in bi.actions) + + const gi = bi.actions["gen-integrity"] + assert("run" in gi) + assert("description" in gi) + assert("arguments" in gi) + + const gia = gi.arguments + assert("_" in gia) + assert("keyfile" in gia) + assert("algorithm" in gia) + assert("output" in gia) + assert("extension" in gia) + + assert("name" in gia._) + assert("description" in gia._) + + assert("description" in gia.keyfile) + assert("string" in gia.keyfile) + assert(!("collect" in gia.keyfile)) + assert(gia.keyfile.string) + + assert("description" in gia.algorithm) + assert("collect" in gia.algorithm) + assert(gia.algorithm.collect) + assert("string" in gia.algorithm) + assert(gia.algorithm.string) + + assert("description" in gia.output) + assert(!("collect" in gia.output)) + assert("string" in gia.output) + assert(gia.output.string) + + assert("description" in gia.extension) + assert(!("collect" in gia.extension)) + assert("string" in gia.extension) + assert(gia.extension.string) +}); + +// this is a separate test in order to catch any changing defaults +Deno.test("gen-integrity action defaults", async () => { + const bi = await import('../../plugins/signed-integrity/cli.js') + const gia = bi.actions["gen-integrity"].arguments + assert("default" in gia.algorithm) + assert(gia.algorithm.default == "SHA-256") + assert("default" in gia.output) + assert(gia.output.default == "json") + assert("default" in gia.extension) + assert(gia.extension.default == ".integrity") +}); + +Deno.test("gen-integrity verifies arguments are sane", async () => { + const bi = await import('../../plugins/signed-integrity/cli.js') + const gi = bi.actions["gen-integrity"] + assertRejects(gi.run, Error, "Expected non-empty list of paths to process.") + assertRejects(async ()=>{ + await gi.run(['no-such-file']) + }, Error, "No keyfile provided.") + assertRejects(async ()=>{ + await gi.run(['no-such-file'], 'irrelevant') + }, Error, "No such file or directory") + assertRejects(async ()=>{ + await gi.run(['irrelevant'], 'irrelevant', []) + }, Error, "Expected non-empty list of algorithms to use.") + assertRejects(async ()=>{ + await gi.run(['irrelevant'], 'irrelevant', ['SHA-384'], false) + }, Error, "Expected 'json', 'text', or 'files' as output type.") + assertRejects(async ()=>{ + await gi.run(['irrelevant'], 'irrelevant', ['SHA-384'], 'files', false) + }, Error, "No extension provided.") + // extension is ignored even if incorrect when output is not 'files' + assertRejects(async ()=>{ + await gi.run(['irrelevant'], 'irrelevant', ['SHA-384'], 'text', false) + }, Error, "Failed to load private key from 'irrelevant': No such file or directory") + assertRejects(async ()=>{ + await gi.run(['irrelevant'], 'irrelevant', ['SHA-384'], 'json', false) + }, Error, "Failed to load private key from 'irrelevant': No such file or directory") +}); + +Deno.test("gen-integrity handles paths in a sane way", async () => { + const bi = await import('../../plugins/signed-integrity/cli.js') + const gi = bi.actions["gen-integrity"] + assertRejects(async ()=>{ + await gi.run(['./'], 'non-existent') + }, Error, "Failed to load private key from 'non-existent'") + assertEquals( + await gi.run(['./'], './__denotests__/mocks/keyfile.json'), + '{}' + ) + /*assertEquals( + )await gi.run(['./__denotests__/mocks/hello.txt'], './__denotests__/mocks/keyfile.json'), + '{}' + )*/ + //assertEquals(await gi.run(['./__denotests__/mocks/hello.txt']), '{"./__denotests__/mocks/hello.txt":["sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek="]}') + //assertEquals(await gi.run(['./', './__denotests__/mocks/hello.txt']), '{"./__denotests__/mocks/hello.txt":["sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek="]}') +}); + +/*Deno.test("gen-integrity handles algos argument in a sane way", async () => { + const bi = await import('../../plugins/signed-integrity/cli.js') + const gi = bi.actions["gen-integrity"] + assertRejects(async ()=>{ + await gi.run(['./__denotests__/mocks/hello.txt'], ['BAD-ALG']) + }, Error, 'Unrecognized algorithm name') + assertEquals(await gi.run(['./__denotests__/mocks/hello.txt'], ['SHA-256']), '{"./__denotests__/mocks/hello.txt":["sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek="]}') + assertEquals(await gi.run(['./__denotests__/mocks/hello.txt'], ['SHA-384']), '{"./__denotests__/mocks/hello.txt":["sha384-/b2OdaZ/KfcBpOBAOF4uI5hjA+oQI5IRr5B/y7g1eLPkF8txzmRu/QgZ3YwIjeG9"]}') + assertEquals(await gi.run(['./__denotests__/mocks/hello.txt'], ['SHA-512']), '{"./__denotests__/mocks/hello.txt":["sha512-MJ7MSJwS1utMxA9QyQLytNDtd+5RGnx6m808qG1M2G+YndNbxf9JlnDaNCVbRbDP2DDoH2Bdz33FVC6TrpzXbw=="]}') + assertEquals(await gi.run(['./__denotests__/mocks/hello.txt'], ['SHA-256', 'SHA-384', 'SHA-512']), '{"./__denotests__/mocks/hello.txt":["sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek=","sha384-/b2OdaZ/KfcBpOBAOF4uI5hjA+oQI5IRr5B/y7g1eLPkF8txzmRu/QgZ3YwIjeG9","sha512-MJ7MSJwS1utMxA9QyQLytNDtd+5RGnx6m808qG1M2G+YndNbxf9JlnDaNCVbRbDP2DDoH2Bdz33FVC6TrpzXbw=="]}') +}); + +Deno.test("gen-integrity handles output argument in a sane way", async () => { + const bi = await import('../../plugins/signed-integrity/cli.js') + const gi = bi.actions["gen-integrity"] + assertEquals(await gi.run(['./__denotests__/mocks/hello.txt'], ['SHA-256'], 'text'), './__denotests__/mocks/hello.txt: sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek=\n') + assertEquals(await gi.run(['./__denotests__/mocks/hello.txt'], ['SHA-384'], 'text'), './__denotests__/mocks/hello.txt: sha384-/b2OdaZ/KfcBpOBAOF4uI5hjA+oQI5IRr5B/y7g1eLPkF8txzmRu/QgZ3YwIjeG9\n') + assertEquals(await gi.run(['./__denotests__/mocks/hello.txt'], ['SHA-512'], 'text'), './__denotests__/mocks/hello.txt: sha512-MJ7MSJwS1utMxA9QyQLytNDtd+5RGnx6m808qG1M2G+YndNbxf9JlnDaNCVbRbDP2DDoH2Bdz33FVC6TrpzXbw==\n') + assertEquals(await gi.run(['./__denotests__/mocks/hello.txt'], ['SHA-256', 'SHA-384', 'SHA-512'], 'text'), './__denotests__/mocks/hello.txt: sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek= sha384-/b2OdaZ/KfcBpOBAOF4uI5hjA+oQI5IRr5B/y7g1eLPkF8txzmRu/QgZ3YwIjeG9 sha512-MJ7MSJwS1utMxA9QyQLytNDtd+5RGnx6m808qG1M2G+YndNbxf9JlnDaNCVbRbDP2DDoH2Bdz33FVC6TrpzXbw==\n') +}); +*/ From a6985e85219945d3781528fe4de0fcbefbbfb8e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27rysiek=27=20Wo=C5=BAniak?= Date: Thu, 15 Dec 2022 16:56:50 +0000 Subject: [PATCH 36/38] cli: more tests for signed-integrity cli done (ref. #66) --- .../plugins/signed-integrity.test.js | 178 ++++++++++++++++-- plugins/basic-integrity/cli.js | 2 - plugins/signed-integrity/cli.js | 35 ++-- 3 files changed, 175 insertions(+), 40 deletions(-) diff --git a/__denotests__/plugins/signed-integrity.test.js b/__denotests__/plugins/signed-integrity.test.js index fd74707..b4a9990 100644 --- a/__denotests__/plugins/signed-integrity.test.js +++ b/__denotests__/plugins/signed-integrity.test.js @@ -2,9 +2,85 @@ import { assert, assertThrows, assertRejects, - assertEquals + assertEquals, + assertStringIncludes } from "https://deno.land/std@0.167.0/testing/asserts.ts"; +// this needs to be the same as the pubkey in: +// ./__denotests__/mocks/keyfile.json +var pubkey = { + "kty": "EC", + "crv": "P-384", + "alg": "ES384", + "x": "rrFawYTuFo8ZjoDxaztUU-c_RAwjw1Y9Tp3j4nH4WsY2Zlizf40Mvz_0BUkVVZCw", + "y": "HaFct6PVK2CQ7ZT2SHClnN-knmGfjY_DFwc6qrAu1s0DFZ8fEUuNdmkTlj9T4NQw", + "key_ops": [ + "verify" + ], + "ext": true +} + +/** + * helper function — decode a b64url-encoded string + */ +let b64urlDecode = (data) => { + data = data.replace(/_/g, '/').replace(/-/g, '+') + data += '='.repeat((4 - (data.length % 4)) % 4) + return atob(data) +} + +/** + * helper function — verify a signed JWT using a provided key + */ +let verifySignedJWT = async (jwt, key) => { + + // working in sections + jwt = jwt.split('.') + + // sections cannot be empty + if ( (jwt[0].length == 0) || (jwt[1].length == 0) || (jwt[2].length == 0) ) { + throw new Error('JWT is invalid, one or more sections are empty.') + } + + // load the key + key = await crypto.subtle.importKey( + "jwk", + key, + { + name: 'ECDSA', + namedCurve: 'P-384' + }, + true, + ['verify'] + ) + + // checking the header + var header = JSON.parse(b64urlDecode(jwt[0])) + if (!("alg" in header) || (header.alg != "ES384")) { + throw new Error('Expected header to contain alg field set to "ES384"') + } + + // get the signature in a usable format + var signature = Uint8Array + .from( + Array + .from(b64urlDecode(jwt[2])) + .map(e=>e.charCodeAt(0)) + ).buffer + + // return the result of signature verification + return await crypto.subtle.verify( + { + name: "ECDSA", + hash: {name: "SHA-384"} + }, + key, + signature, + new TextEncoder("utf-8").encode(jwt[0] + '.' + jwt[1]) + ) +} + + Deno.test("plugin loads", async () => { const bi = await import('../../plugins/signed-integrity/cli.js') assert("name" in bi) @@ -94,42 +170,104 @@ Deno.test("gen-integrity verifies arguments are sane", async () => { }, Error, "Failed to load private key from 'irrelevant': No such file or directory") }); -Deno.test("gen-integrity handles paths in a sane way", async () => { +Deno.test("gen-integrity handles paths correctly", async () => { const bi = await import('../../plugins/signed-integrity/cli.js') const gi = bi.actions["gen-integrity"] assertRejects(async ()=>{ await gi.run(['./'], 'non-existent') }, Error, "Failed to load private key from 'non-existent'") + assertRejects(async ()=>{ + await gi.run(['./'], './') + }, Error, "ailed to load private key from './': Is a directory") assertEquals( await gi.run(['./'], './__denotests__/mocks/keyfile.json'), '{}' ) - /*assertEquals( - )await gi.run(['./__denotests__/mocks/hello.txt'], './__denotests__/mocks/keyfile.json'), - '{}' - )*/ - //assertEquals(await gi.run(['./__denotests__/mocks/hello.txt']), '{"./__denotests__/mocks/hello.txt":["sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek="]}') - //assertEquals(await gi.run(['./', './__denotests__/mocks/hello.txt']), '{"./__denotests__/mocks/hello.txt":["sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek="]}') + assertStringIncludes( + await gi.run(['./__denotests__/mocks/hello.txt'], './__denotests__/mocks/keyfile.json'), + '"./__denotests__/mocks/hello.txt":"eyJhbGciOiAiRVMzODQifQ.eyJpbnRlZ3JpdHkiOiAic2hhMjU2LXVVMG51Wk5OUGdpbExsTFgybjJyK3NTRTcrTjZVNER1a0lqM3JPTHZ6ZWs9In0.' + ) }); -/*Deno.test("gen-integrity handles algos argument in a sane way", async () => { +Deno.test("gen-integrity handles algos argument correctly", async () => { const bi = await import('../../plugins/signed-integrity/cli.js') const gi = bi.actions["gen-integrity"] assertRejects(async ()=>{ - await gi.run(['./__denotests__/mocks/hello.txt'], ['BAD-ALG']) + await gi.run(['./__denotests__/mocks/hello.txt'], './__denotests__/mocks/keyfile.json', ['BAD-ALG']) }, Error, 'Unrecognized algorithm name') - assertEquals(await gi.run(['./__denotests__/mocks/hello.txt'], ['SHA-256']), '{"./__denotests__/mocks/hello.txt":["sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek="]}') - assertEquals(await gi.run(['./__denotests__/mocks/hello.txt'], ['SHA-384']), '{"./__denotests__/mocks/hello.txt":["sha384-/b2OdaZ/KfcBpOBAOF4uI5hjA+oQI5IRr5B/y7g1eLPkF8txzmRu/QgZ3YwIjeG9"]}') - assertEquals(await gi.run(['./__denotests__/mocks/hello.txt'], ['SHA-512']), '{"./__denotests__/mocks/hello.txt":["sha512-MJ7MSJwS1utMxA9QyQLytNDtd+5RGnx6m808qG1M2G+YndNbxf9JlnDaNCVbRbDP2DDoH2Bdz33FVC6TrpzXbw=="]}') - assertEquals(await gi.run(['./__denotests__/mocks/hello.txt'], ['SHA-256', 'SHA-384', 'SHA-512']), '{"./__denotests__/mocks/hello.txt":["sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek=","sha384-/b2OdaZ/KfcBpOBAOF4uI5hjA+oQI5IRr5B/y7g1eLPkF8txzmRu/QgZ3YwIjeG9","sha512-MJ7MSJwS1utMxA9QyQLytNDtd+5RGnx6m808qG1M2G+YndNbxf9JlnDaNCVbRbDP2DDoH2Bdz33FVC6TrpzXbw=="]}') + + // helper function + let getGeneratedTestIntegrity = async (algos) => { + let integrity = JSON.parse(await gi.run( + ['./__denotests__/mocks/hello.txt'], + './__denotests__/mocks/keyfile.json', + algos) + ) + integrity = b64urlDecode(integrity["./__denotests__/mocks/hello.txt"].split('.')[1]) + return integrity + } + + assertEquals(await getGeneratedTestIntegrity(['SHA-256']), '{"integrity": "sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek="}') + assertEquals(await getGeneratedTestIntegrity(['SHA-384']), '{"integrity": "sha384-/b2OdaZ/KfcBpOBAOF4uI5hjA+oQI5IRr5B/y7g1eLPkF8txzmRu/QgZ3YwIjeG9"}') + assertEquals(await getGeneratedTestIntegrity(['SHA-512']), '{"integrity": "sha512-MJ7MSJwS1utMxA9QyQLytNDtd+5RGnx6m808qG1M2G+YndNbxf9JlnDaNCVbRbDP2DDoH2Bdz33FVC6TrpzXbw=="}') + assertEquals(await getGeneratedTestIntegrity(['SHA-256', 'SHA-384', 'SHA-512']), '{"integrity": "sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek= sha384-/b2OdaZ/KfcBpOBAOF4uI5hjA+oQI5IRr5B/y7g1eLPkF8txzmRu/QgZ3YwIjeG9 sha512-MJ7MSJwS1utMxA9QyQLytNDtd+5RGnx6m808qG1M2G+YndNbxf9JlnDaNCVbRbDP2DDoH2Bdz33FVC6TrpzXbw=="}') }); -Deno.test("gen-integrity handles output argument in a sane way", async () => { +Deno.test("gen-integrity text output is correct", async () => { const bi = await import('../../plugins/signed-integrity/cli.js') const gi = bi.actions["gen-integrity"] - assertEquals(await gi.run(['./__denotests__/mocks/hello.txt'], ['SHA-256'], 'text'), './__denotests__/mocks/hello.txt: sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek=\n') - assertEquals(await gi.run(['./__denotests__/mocks/hello.txt'], ['SHA-384'], 'text'), './__denotests__/mocks/hello.txt: sha384-/b2OdaZ/KfcBpOBAOF4uI5hjA+oQI5IRr5B/y7g1eLPkF8txzmRu/QgZ3YwIjeG9\n') - assertEquals(await gi.run(['./__denotests__/mocks/hello.txt'], ['SHA-512'], 'text'), './__denotests__/mocks/hello.txt: sha512-MJ7MSJwS1utMxA9QyQLytNDtd+5RGnx6m808qG1M2G+YndNbxf9JlnDaNCVbRbDP2DDoH2Bdz33FVC6TrpzXbw==\n') - assertEquals(await gi.run(['./__denotests__/mocks/hello.txt'], ['SHA-256', 'SHA-384', 'SHA-512'], 'text'), './__denotests__/mocks/hello.txt: sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek= sha384-/b2OdaZ/KfcBpOBAOF4uI5hjA+oQI5IRr5B/y7g1eLPkF8txzmRu/QgZ3YwIjeG9 sha512-MJ7MSJwS1utMxA9QyQLytNDtd+5RGnx6m808qG1M2G+YndNbxf9JlnDaNCVbRbDP2DDoH2Bdz33FVC6TrpzXbw==\n') + + let getGeneratedTestIntegrity = async (algos) => { + let result = await gi.run( + ['./__denotests__/mocks/hello.txt'], + './__denotests__/mocks/keyfile.json', + algos, + 'text' + ) + result = result.split(' ') + return [result[0], b64urlDecode(result[1].split('.')[1])] + } + + assertEquals( + await getGeneratedTestIntegrity(['SHA-256']), + [ + "./__denotests__/mocks/hello.txt:", + '{"integrity": "sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek="}' + ] + ) + assertEquals( + await getGeneratedTestIntegrity(['SHA-384']), + [ + "./__denotests__/mocks/hello.txt:", + '{"integrity": "sha384-/b2OdaZ/KfcBpOBAOF4uI5hjA+oQI5IRr5B/y7g1eLPkF8txzmRu/QgZ3YwIjeG9"}' + ] + ) + assertEquals( + await getGeneratedTestIntegrity(['SHA-512']), + [ + "./__denotests__/mocks/hello.txt:", + '{"integrity": "sha512-MJ7MSJwS1utMxA9QyQLytNDtd+5RGnx6m808qG1M2G+YndNbxf9JlnDaNCVbRbDP2DDoH2Bdz33FVC6TrpzXbw=="}' + ] + ) + + + assertEquals( + await getGeneratedTestIntegrity(['SHA-256', 'SHA-384', 'SHA-512']), + [ + "./__denotests__/mocks/hello.txt:", + '{"integrity": "sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek= sha384-/b2OdaZ/KfcBpOBAOF4uI5hjA+oQI5IRr5B/y7g1eLPkF8txzmRu/QgZ3YwIjeG9 sha512-MJ7MSJwS1utMxA9QyQLytNDtd+5RGnx6m808qG1M2G+YndNbxf9JlnDaNCVbRbDP2DDoH2Bdz33FVC6TrpzXbw=="}' + ] + ) }); -*/ + +// TODO: "files" output mode, which will require mocking file writing routines + +Deno.test("gen-integrity signs the data correctly", async () => { + const bi = await import('../../plugins/signed-integrity/cli.js') + const gi = bi.actions["gen-integrity"] + let jwt = JSON.parse(await gi.run(['./__denotests__/mocks/hello.txt'], './__denotests__/mocks/keyfile.json')) + assert( + await verifySignedJWT( + jwt['./__denotests__/mocks/hello.txt'], + pubkey)) +}) diff --git a/plugins/basic-integrity/cli.js b/plugins/basic-integrity/cli.js index 83ad926..93e3b0c 100644 --- a/plugins/basic-integrity/cli.js +++ b/plugins/basic-integrity/cli.js @@ -47,8 +47,6 @@ let getFileIntegrity = async (path, algos) => { // are we working with a file? if (fileInfo.isFile) { - //console.log(`+-- reading: ${path}`) - // initialize content = new Uint8Array() var buf = new Uint8Array(1000); diff --git a/plugins/signed-integrity/cli.js b/plugins/signed-integrity/cli.js index e6c2328..d0063a3 100644 --- a/plugins/signed-integrity/cli.js +++ b/plugins/signed-integrity/cli.js @@ -106,6 +106,7 @@ let getPubkey = async (keyfile) => { let getFileIntegrity = async (path, algos) => { var result = [] + var content = false // open the file and get some info on it const file = await Deno.open( @@ -117,10 +118,8 @@ let getFileIntegrity = async (path, algos) => { // are we working with a file? if (fileInfo.isFile) { - //console.log(`+-- reading: ${path}`) - // initialize - var content = new Uint8Array() + content = new Uint8Array() var buf = new Uint8Array(1000); // read the first batch @@ -145,22 +144,8 @@ let getFileIntegrity = async (path, algos) => { // read some more numread = file.readSync(buf); } - //console.log(' +-- done.') - - //console.log('+-- calculating digests') - for (const algo of algos) { - //console.log(` +-- ${algo}`) - var digest = algo.toLowerCase().replace('-', '') + '-' + binToBase64(await crypto.subtle.digest(algo, content)) - //console.log(digest) - result.push(digest) - } - //console.log(`+-- file done: ${path}`) - - // we are not working with a file - } else { - result = false; } - + // putting this in a try-catch block as the file // is apparently being auto-closed? // https://issueantenna.com/repo/denoland/deno/issues/15442 @@ -168,6 +153,20 @@ let getFileIntegrity = async (path, algos) => { await file.close(); } catch (BadResource) {} + // did we get any content? + if (typeof content != "boolean") { + for (const algo of algos) { + //console.log(` +-- ${algo}`) + var digest = algo.toLowerCase().replace('-', '') + '-' + binToBase64(await crypto.subtle.digest(algo, content)) + //console.log(digest) + result.push(digest) + } + // no content means not a file + } else { + // so no result + result = false + } + // return the result return result } From 7858687e5bb4908a0b47ea62a9d581a16de6cf3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27rysiek=27=20Wo=C5=BAniak?= Date: Thu, 15 Dec 2022 17:29:36 +0000 Subject: [PATCH 37/38] cli: test for signed-integrity gen-keypair action (ref. #66) --- .../plugins/signed-integrity.test.js | 63 ++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/__denotests__/plugins/signed-integrity.test.js b/__denotests__/plugins/signed-integrity.test.js index b4a9990..3181628 100644 --- a/__denotests__/plugins/signed-integrity.test.js +++ b/__denotests__/plugins/signed-integrity.test.js @@ -3,7 +3,8 @@ import { assertThrows, assertRejects, assertEquals, - assertStringIncludes + assertStringIncludes, + assertObjectMatch } from "https://deno.land/std@0.167.0/testing/asserts.ts"; // this needs to be the same as the pubkey in: @@ -271,3 +272,63 @@ Deno.test("gen-integrity signs the data correctly", async () => { jwt['./__denotests__/mocks/hello.txt'], pubkey)) }) + +Deno.test("get-pubkey works correctly", async () => { + const bi = await import('../../plugins/signed-integrity/cli.js') + const gp = bi.actions["get-pubkey"] + assertRejects(gp.run, Error, "No keyfile provided.") + assertRejects(async ()=>{ + await gp.run('no-such-file') + }, Error, "No such file or directory") + assertRejects(async ()=>{ + await gp.run(['no-such-file']) + }, Error, "No such file or directory") + assertEquals( + await gp.run('./__denotests__/mocks/keyfile.json'), + '{"kty":"EC","crv":"P-384","alg":"ES384","x":"rrFawYTuFo8ZjoDxaztUU-c_RAwjw1Y9Tp3j4nH4WsY2Zlizf40Mvz_0BUkVVZCw","y":"HaFct6PVK2CQ7ZT2SHClnN-knmGfjY_DFwc6qrAu1s0DFZ8fEUuNdmkTlj9T4NQw","key_ops":["verify"],"ext":true}' + ) + assertEquals( + await gp.run(['./__denotests__/mocks/keyfile.json', 'irrelevant']), + '{"kty":"EC","crv":"P-384","alg":"ES384","x":"rrFawYTuFo8ZjoDxaztUU-c_RAwjw1Y9Tp3j4nH4WsY2Zlizf40Mvz_0BUkVVZCw","y":"HaFct6PVK2CQ7ZT2SHClnN-knmGfjY_DFwc6qrAu1s0DFZ8fEUuNdmkTlj9T4NQw","key_ops":["verify"],"ext":true}' + ) +}); + +Deno.test("gen-keypair works correctly", async () => { + const bi = await import('../../plugins/signed-integrity/cli.js') + const gk = bi.actions["gen-keypair"] + const keypair = JSON.parse(await gk.run()) + assert('privateKey' in keypair) + assert('x' in keypair.privateKey) + assert('y' in keypair.privateKey) + assert('d' in keypair.privateKey) + assertObjectMatch( + keypair.privateKey, + { + kty: "EC", + crv: "P-384", + alg: "ES384", + key_ops: [ + "sign" + ], + ext: true + } + ) + assert('publicKey' in keypair) + assert('x' in keypair.publicKey) + assert('y' in keypair.publicKey) + assert(!('d' in keypair.publicKey)) + assertObjectMatch( + keypair.publicKey, + { + kty: "EC", + crv: "P-384", + alg: "ES384", + key_ops: [ + "verify" + ], + ext: true + } + ) + assert((keypair.privateKey.x == keypair.publicKey.x)) + assert((keypair.privateKey.y == keypair.publicKey.y)) +}); From aff6709fe6ef12867391ec058caf637179a78e01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27rysiek=27=20Wo=C5=BAniak?= Date: Fri, 16 Dec 2022 17:51:35 +0000 Subject: [PATCH 38/38] jest should now ignore deno-related files and tests when calculating coverage --- package.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/package.json b/package.json index ece6b15..789739f 100644 --- a/package.json +++ b/package.json @@ -2,6 +2,8 @@ "jest": { "collectCoverageFrom": [ "**/*.js", + "!**/plugins/**/cli.js", + "!**/__denotests__/**", "!**/node_modules/**", "!**/lib/**" ],