From 7d96876aca591f0aa65ed45fcad20a9397af4650 Mon Sep 17 00:00:00 2001 From: Nolan Lawson Date: Sat, 6 Mar 2021 09:06:42 -0800 Subject: [PATCH] chore: break up circle CI tasks (#1983) --- .circleci/config.yml | 10 ++++-- bin/install-mastodon.js | 77 +++++++++++++++++++++++++++++++++++++++++ bin/mastodon-config.js | 15 ++++++++ bin/run-mastodon.js | 72 ++------------------------------------ package.json | 2 ++ 5 files changed, 104 insertions(+), 72 deletions(-) create mode 100644 bin/install-mastodon.js diff --git a/.circleci/config.yml b/.circleci/config.yml index f74a8176..dc537a41 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -47,6 +47,9 @@ jobs: - run: name: Unit tests command: yarn test-unit + - run: + name: Install mastodon + command: yarn install-mastodon - run: name: Wait for postgres to be ready command: | @@ -67,11 +70,12 @@ jobs: sleep 1 done echo Failed waiting for redis && exit 1 + - run: + name: Build + command: yarn build - run: name: Integration tests - command: | - node -v - yarn test + command: yarn test-in-ci - save_cache: name: Save yarn cache key: yarn-v1-{{ checksum "yarn.lock" }} diff --git a/bin/install-mastodon.js b/bin/install-mastodon.js new file mode 100644 index 00000000..856a7221 --- /dev/null +++ b/bin/install-mastodon.js @@ -0,0 +1,77 @@ +import { promisify } from 'util' +import childProcessPromise from 'child-process-promise' +import path from 'path' +import fs from 'fs' +import { DB_NAME, DB_PASS, DB_USER, mastodonDir, env } from './mastodon-config' +import mkdirp from 'mkdirp' + +const exec = childProcessPromise.exec +const stat = promisify(fs.stat) +const writeFile = promisify(fs.writeFile) +const dir = __dirname + +async function setupMastodonDatabase () { + console.log('Setting up mastodon database...') + try { + await exec(`psql -d template1 -c "CREATE USER ${DB_USER} WITH PASSWORD '${DB_PASS}' CREATEDB;"`) + } catch (e) { /* ignore */ } + try { + await exec(`dropdb -h 127.0.0.1 -U ${DB_USER} -w ${DB_NAME}`, { + cwd: mastodonDir, + env: Object.assign({ PGPASSWORD: DB_PASS }, process.env) + }) + } catch (e) { /* ignore */ } + await exec(`createdb -h 127.0.0.1 -U ${DB_USER} -w ${DB_NAME}`, { + cwd: mastodonDir, + env: Object.assign({ PGPASSWORD: DB_PASS }, process.env) + }) + + const dumpFile = path.join(dir, '../tests/fixtures/dump.sql') + await exec(`psql -h 127.0.0.1 -U ${DB_USER} -w -d ${DB_NAME} -f "${dumpFile}"`, { + cwd: mastodonDir, + env: Object.assign({ PGPASSWORD: DB_PASS }, process.env) + }) + + const tgzFile = path.join(dir, '../tests/fixtures/system.tgz') + const systemDir = path.join(mastodonDir, 'public/system') + await mkdirp(systemDir) + await exec(`tar -xzf "${tgzFile}"`, { cwd: systemDir }) +} + +async function installMastodonDependencies () { + const cwd = mastodonDir + const installCommands = [ + 'gem update --system', + 'gem install bundler foreman', + 'bundle config set --local frozen \'true\'', + 'bundle install', + 'yarn --pure-lockfile' + ] + + const installedFile = path.join(mastodonDir, 'installed.txt') + try { + await stat(installedFile) + console.log('Already installed Mastodon dependencies') + } catch (e) { + console.log('Installing Mastodon dependencies...') + for (const cmd of installCommands) { + console.log(cmd) + await exec(cmd, { cwd, env }) + } + await writeFile(installedFile, '', 'utf8') + } + await exec('bundle exec rails db:migrate', { cwd, env }) +} + +export default async function installMastodon () { + console.log('Installing Mastodon...') + await setupMastodonDatabase() + await installMastodonDependencies() +} + +if (require.main === module) { + installMastodon().catch(err => { + console.error(err) + process.exit(1) + }) +} diff --git a/bin/mastodon-config.js b/bin/mastodon-config.js index 1b846482..88ba0b17 100644 --- a/bin/mastodon-config.js +++ b/bin/mastodon-config.js @@ -1,3 +1,5 @@ +import path from 'path' + export const DB_NAME = 'pinafore_development' export const DB_USER = 'pinafore' export const DB_PASS = 'pinafore' @@ -18,3 +20,16 @@ DB_PASS=${DB_PASS} // Need a Ruby version that CircleCI bundles with Node v12, not Node v14 which doesn't // work for streaming export const RUBY_VERSION = '2.6.6' + +export const mastodonDir = path.join(__dirname, '../mastodon') + +export const env = Object.assign({}, process.env, { + RAILS_ENV: 'development', + NODE_ENV: 'development', + BUNDLE_PATH: path.join(mastodonDir, 'vendor/bundle'), + DB_NAME, + DB_USER, + DB_PASS, + DB_HOST, + DB_PORT +}) diff --git a/bin/run-mastodon.js b/bin/run-mastodon.js index b80a533f..55b4b802 100644 --- a/bin/run-mastodon.js +++ b/bin/run-mastodon.js @@ -1,84 +1,18 @@ import { restoreMastodonData } from './restore-mastodon-data' -import { promisify } from 'util' import childProcessPromise from 'child-process-promise' -import path from 'path' import fs from 'fs' import { waitForMastodonUiToStart, waitForMastodonApiToStart } from './wait-for-mastodon-to-start' -import mkdirp from 'mkdirp' import cloneMastodon from './clone-mastodon' -import { DB_USER, DB_PASS, DB_NAME, DB_HOST, DB_PORT } from './mastodon-config' +import installMastodon from './install-mastodon' +import { mastodonDir, env } from './mastodon-config' -const exec = childProcessPromise.exec const spawn = childProcessPromise.spawn -const stat = promisify(fs.stat) -const writeFile = promisify(fs.writeFile) -const dir = __dirname -const mastodonDir = path.join(dir, '../mastodon') let childProc -async function setupMastodonDatabase () { - console.log('Setting up mastodon database...') - try { - await exec(`psql -d template1 -c "CREATE USER ${DB_USER} WITH PASSWORD '${DB_PASS}' CREATEDB;"`) - } catch (e) { /* ignore */ } - try { - await exec(`dropdb -h 127.0.0.1 -U ${DB_USER} -w ${DB_NAME}`, { - cwd: mastodonDir, - env: Object.assign({ PGPASSWORD: DB_PASS }, process.env) - }) - } catch (e) { /* ignore */ } - await exec(`createdb -h 127.0.0.1 -U ${DB_USER} -w ${DB_NAME}`, { - cwd: mastodonDir, - env: Object.assign({ PGPASSWORD: DB_PASS }, process.env) - }) - - const dumpFile = path.join(dir, '../tests/fixtures/dump.sql') - await exec(`psql -h 127.0.0.1 -U ${DB_USER} -w -d ${DB_NAME} -f "${dumpFile}"`, { - cwd: mastodonDir, - env: Object.assign({ PGPASSWORD: DB_PASS }, process.env) - }) - - const tgzFile = path.join(dir, '../tests/fixtures/system.tgz') - const systemDir = path.join(mastodonDir, 'public/system') - await mkdirp(systemDir) - await exec(`tar -xzf "${tgzFile}"`, { cwd: systemDir }) -} - async function runMastodon () { console.log('Running mastodon...') - const env = Object.assign({}, process.env, { - RAILS_ENV: 'development', - NODE_ENV: 'development', - BUNDLE_PATH: path.join(mastodonDir, 'vendor/bundle'), - DB_NAME, - DB_USER, - DB_PASS, - DB_HOST, - DB_PORT - }) const cwd = mastodonDir - const installCommands = [ - 'gem update --system', - 'gem install bundler foreman', - 'bundle config set --local frozen \'true\'', - 'bundle install', - 'yarn --pure-lockfile' - ] - - const installedFile = path.join(mastodonDir, 'installed.txt') - try { - await stat(installedFile) - console.log('Already installed Mastodon') - } catch (e) { - console.log('Installing Mastodon...') - for (const cmd of installCommands) { - console.log(cmd) - await exec(cmd, { cwd, env }) - } - await writeFile(installedFile, '', 'utf8') - } - await exec('bundle exec rails db:migrate', { cwd, env }) const promise = spawn('foreman', ['start'], { cwd, env }) // don't bother writing to mastodon.log in CI; we can't read the file anyway const logFile = process.env.CIRCLECI ? '/dev/null' : 'mastodon.log' @@ -96,7 +30,7 @@ async function runMastodon () { async function main () { await cloneMastodon() - await setupMastodonDatabase() + await installMastodon() await runMastodon() await waitForMastodonApiToStart() await restoreMastodonData() diff --git a/package.json b/package.json index ec20a6c4..654a712d 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "build-template-html-watch": "node -r esm ./bin/build-template-html.js --watch", "build-assets": "node -r esm ./bin/build-assets.js", "clone-mastodon": "node -r esm ./bin/clone-mastodon.js", + "install-mastodon": "node -r esm ./bin/install-mastodon.js", "run-mastodon": "node -r esm ./bin/run-mastodon.js", "test": "cross-env BROWSER=chrome:headless run-s test-browser", "test-browser": "run-p --race run-mastodon build-and-start test-mastodon", @@ -26,6 +27,7 @@ "testcafe-suite0": "cross-env-shell testcafe -c 4 $BROWSER tests/spec/0*", "testcafe-suite1": "cross-env-shell testcafe $BROWSER tests/spec/1*", "test-unit": "NODE_ENV=test mocha -r esm -r bin/browser-shim.js tests/unit/", + "test-in-ci": "cross-env BROWSER=chrome:headless run-p --race run-mastodon start test-mastodon", "wait-for-mastodon-to-start": "node -r esm bin/wait-for-mastodon-to-start.js", "wait-for-mastodon-data": "node -r esm bin/wait-for-mastodon-data.js", "backup-mastodon-data": "./bin/backup-mastodon-data.sh",