diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 197f0c3..9bb218a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,13 +1,15 @@ -name: Test/Release +name: Test-Release on: [push] jobs: - test_maybe_release: - name: Test (then release if new version) + tests: + name: "Tests (webextension, interfacer: unit, tty, http-server)" runs-on: ubuntu-latest env: GOPATH: ${{ github.workspace }} GOBIN: ${{ github.workspace }}/bin + outputs: + is_new_version: ${{ steps.check_versions.outputs.is_new_version }} steps: - name: Checkout uses: actions/checkout@v3 @@ -16,15 +18,15 @@ jobs: - name: Setup go uses: actions/setup-go@v3 with: - go-version: 1.18.x + go-version-file: 'interfacer/go.mod' - name: Setup node uses: actions/setup-node@v3 with: - node-version: 16 + node-version-file: '.nvmrc' - name: Install Firefox uses: browser-actions/setup-firefox@latest with: - firefox-version: 102.0.1 # TODO: Use same version in Dockerfile + firefox-version: 102.0.1 - run: firefox --version # Web extension tests @@ -33,14 +35,13 @@ jobs: - name: Web extension tests run: npm test working-directory: ./webext - - run: npm run build_webextension - working-directory: ./webext # Interfacer tests - - name: Build - run: go build ./cmd/browsh - working-directory: ./interfacer - + - name: Interfacer tests setup + run: | + touch interfacer/src/browsh/browsh.xpi + cd webext + npm run build:dev - name: Unit tests run: go test -v $(find src/browsh -name '*.go' | grep -v windows) working-directory: ./interfacer @@ -51,19 +52,64 @@ jobs: if: ${{ failure() }} run: cat ./interfacer/test/tty/debug.log || echo "No log file" - name: HTTP Server tests - run: go test test/http-server/*.go -v -ginkgo.slowSpecThreshold=30 -ginkgo.flakeAttempts=3 - working-directory: ./interfacer + uses: nick-fields/retry@v2 + with: + max_attempts: 3 + retry_on: error + timeout_minutes: 15 + command: | + cd interfacer + go test test/http-server/*.go -v -ginkgo.slowSpecThreshold=30 -ginkgo.flakeAttempts=3 - name: HTTP Server debug log if: ${{ failure() }} run: cat ./interfacer/test/http-server/debug.log || echo "No log file" - # Release - name: Check for new version id: check_versions run: ./ctl.sh github_actions_output_version_status - - name: Release - if: contains(steps.check_versions.outputs.is_new_version, 'true') + + release: + name: Release + runs-on: ubuntu-latest + needs: tests + if: github.ref == 'refs/heads/master' && contains(needs.tests.outputs.is_new_version, 'true') + env: + GOPATH: ${{ github.workspace }} + GOBIN: ${{ github.workspace }}/bin + MDN_KEY: ${{ secrets.MDN_KEY }} + DOCKER_ACCESS_TOKEN: ${{ secrets.DOCKER_ACCESS_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Setup Deploy Keys + uses: webfactory/ssh-agent@v0.5.4 + with: + # Note that these private keys depend on having an ssh-keygen'd comment with the + # Git remote URL. This is because Github Actions use the *first* matching private + # key and fails if it doesn't match. webfactory/ssh-agent + ssh-private-key: | + ${{ secrets.HOMEBREW_DEPLOY_KEY }} + ${{ secrets.WWW_DEPLOY_KEY }} + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Setup node + uses: actions/setup-node@v3 + with: + node-version-file: '.nvmrc' + - name: Setup go + uses: actions/setup-go@v3 + with: + go-version-file: 'interfacer/go.mod' + - run: npm ci + working-directory: ./webext + - name: Binary Release run: ./ctl.sh release + - name: Push new tag + uses: ad-m/github-push-action@master + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + tags: true - name: Login to Docker Hub uses: docker/login-action@v2 with: @@ -71,5 +117,7 @@ jobs: password: ${{ secrets.DOCKER_ACCESS_TOKEN }} - name: Docker Release run: ./ctl.sh docker_release + - name: Update Homebrew Tap + run: ./ctl.sh update_homebrew_tap_with_new_version - name: Update Browsh Website run: ./ctl.sh update_browsh_website_with_new_version diff --git a/.gitignore b/.gitignore index 539a2ec..c6b3eb2 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,14 @@ webext/node_modules webext/dist/* dist *.xpi + +# This is because of an odd permissions quirk on Github Actions. I can't seem to find a +# way to delete these files in CI, so let's just ignore them. Otherwise Goreleaser complains +# about a dirty working tree. +/pkg +/bin + +# Goreleaser needs to upload the webextension as an extra file in the release. But it doesn't +# like Git to be in a dirty state. Also Goreleaser is run at PWD ./interfacer, so we can't +# reference the webextension with something like ../webext/... +interfacer/browsh-*.xpi diff --git a/Dockerfile b/Dockerfile index 7199d26..7cf07aa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,21 +24,19 @@ ADD .github .github ADD scripts scripts ADD ctl.sh . -# Install Golang +# Install Golang and Browsh ENV GOROOT=/go ENV GOPATH=/go-home ENV PATH=$GOROOT/bin:$GOPATH/bin:$PATH -RUN /build/ctl.sh install_golang +ENV BASE=$GOPATH/src/browsh/interfacer +ADD interfacer $BASE +WORKDIR $BASE +RUN /build/ctl.sh install_golang $BASE +RUN /build/ctl.sh build_browsh_binary $BASE # Install firefox RUN /build/ctl.sh install_firefox -# Build Browsh -ENV BASE=$GOPATH/src/browsh/interfacer -WORKDIR $BASE -ADD interfacer $BASE -RUN /build/ctl.sh build_browsh_binary $BASE - ########################### # Actual final Docker image diff --git a/README.md b/README.md index 0347613..e997bfd 100644 --- a/README.md +++ b/README.md @@ -35,8 +35,8 @@ is running somewhere else on mains electricity. ## Installation -Download a binary from the [releases](https://github.com/browsh-org/browsh/releases) (~7MB). -You will need to have [Firefox 63](https://www.mozilla.org/en-US/firefox/new/) (or higher) already installed. +Download a binary from the [releases](https://github.com/browsh-org/browsh/releases) (~11MB). +You will need to have [Firefox](https://www.mozilla.org/en-US/firefox/new/) already installed. Or download and run the Docker image (~230MB) with: `docker run --rm -it browsh/browsh` diff --git a/ctl.sh b/ctl.sh index bdf143b..816899a 100755 --- a/ctl.sh +++ b/ctl.sh @@ -1,11 +1,12 @@ -#!/bin/env bash +#!/usr/bin/env bash set -e function_to_run=$1 -export PROJECT_ROOT && PROJECT_ROOT=$(git rev-parse --show-toplevel) +export PROJECT_ROOT export GORELEASER_VERSION=1.10.2 -export GOBINDATA_VERSION=3.23.0 + +PROJECT_ROOT=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) function _includes_path { echo "$PROJECT_ROOT"/scripts @@ -26,4 +27,7 @@ if [[ $(type -t "$function_to_run") != function ]]; then fi shift + +pushd "$PROJECT_ROOT" || _panic "$function_to_run" "$@" +popd || _panic diff --git a/goreleaser.yml b/goreleaser.yml index 9e6b865..0762b44 100644 --- a/goreleaser.yml +++ b/goreleaser.yml @@ -57,15 +57,14 @@ nfpms: brews: - name: browsh tap: - owner: browsh-org name: homebrew-browsh - commit_author: - name: Goreleaser Bot care of Github Actions - email: ci@github.com homepage: "https://www.brow.sh" description: "The modern, text-based browser" caveats: "You need Firefox 57 or newer to run Browsh" + # We do the upload manually because Goreleaser doesn't support Deploy Keys and Github + # doesn't support repo-specific Access Tokens 🙄 + skip_upload: true release: extra_files: - - glob: ./webext/dist/web-ext-artifacts/browsh-{{ Env.BROWSH_VERSION }}-an+fx.xpi + - glob: ./browsh-*.xpi diff --git a/interfacer/contrib/setup_firefox.sh b/interfacer/contrib/setup_firefox.sh deleted file mode 100755 index ce3bd17..0000000 --- a/interfacer/contrib/setup_firefox.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash - -set -ex - -if [ ! -f .travis.yml ]; then - PROJECT_ROOT=$(git rev-parse --show-toplevel) -else - PROJECT_ROOT=. -fi - -line=$(grep 'firefox: "' < $PROJECT_ROOT/.travis.yml) -version=$(echo $line | grep -o '".*"' | cut -d " " -f 1 | sed 's/"//g') - -# Firefox is needed both for testing in Travis and embedding in the Docker -# image used by the Browsh as a Service platform. So we need to be able to -# give a specific and consistent version pin. -FIREFOX_VERSION=$version - -mkdir -p $HOME/bin -pushd $HOME/bin -curl -L -o firefox.tar.bz2 https://ftp.mozilla.org/pub/firefox/releases/$FIREFOX_VERSION/linux-x86_64/en-US/firefox-$FIREFOX_VERSION.tar.bz2 -bzip2 -d firefox.tar.bz2 -tar xf firefox.tar -popd - -firefox --version diff --git a/interfacer/src/browsh/firefox.go b/interfacer/src/browsh/firefox.go index 7bb3cbd..06614e6 100644 --- a/interfacer/src/browsh/firefox.go +++ b/interfacer/src/browsh/firefox.go @@ -230,8 +230,6 @@ func firefoxMarionette() { sendFirefoxCommand("WebDriver:NewSession", map[string]interface{}{}) } -// Install the Browsh extension that was bundled with `go-bindata` under -// `webextension.go`. func installWebextension() { data, err := browshXpi.ReadFile("browsh.xpi") if err != nil { diff --git a/interfacer/src/browsh/version.go b/interfacer/src/browsh/version.go index ce25757..3ecc858 100644 --- a/interfacer/src/browsh/version.go +++ b/interfacer/src/browsh/version.go @@ -1,3 +1,3 @@ package browsh -var browshVersion = "1.6.4" +var browshVersion = "1.8.0" diff --git a/scripts/bundling.bash b/scripts/bundling.bash index eb3c538..a46bfcf 100644 --- a/scripts/bundling.bash +++ b/scripts/bundling.bash @@ -1,7 +1,23 @@ #!/usr/bin/env bash -function build_webextension() { - local NODE_BIN=$PROJECT_ROOT/webext/node_modules/.bin +export XPI_PATH="$PROJECT_ROOT"/interfacer/src/browsh/browsh.xpi +export XPI_SOURCE_DIR=$PROJECT_ROOT/webext/dist/web-ext-artifacts +export NODE_BIN=$PROJECT_ROOT/webext/node_modules/.bin +MDN_USER="user:13243312:78" + +function versioned_xpi_file() { + echo "$XPI_SOURCE_DIR/browsh-$(browsh_version).xpi" +} + +# You'll want to use this with `go run ./cmd/browsh --debug --firefox.use-existing` +function build_webextension_watch() { + "$NODE_BIN"/web-ext run \ + --firefox contrib/firefoxheadless.sh \ + --verbose +} + +function build_webextension_production() { + local version && version=$(browsh_version) cd "$PROJECT_ROOT"/webext && "$NODE_BIN"/webpack cd "$PROJECT_ROOT"/webext/dist && rm ./*.map @@ -13,36 +29,63 @@ function build_webextension() { "$NODE_BIN"/web-ext build --overwrite-dest ls -alh web-ext-artifacts - version=$(browsh_version) + webextension_sign + local source_file && source_file=$(versioned_xpi_file) - local source_file - local source_dir=$PROJECT_ROOT/webext/dist/web-ext-artifacts - local bundle_file=$PROJECT_ROOT/interfacer/src/browsh/browsh.xpi + echo "Bundling $source_file to $XPI_PATH" + cp -f "$source_file" "$XPI_PATH" - if [ "$BROWSH_ENV" == "RELEASE" ]; then - # The signed version. There can only be one canonical XPI for each semantic - # version. - source_file="$source_dir/browsh-$version-an+fx.xpi" + echo "Making extra copy for Goreleaser to put in Github release:" + local goreleaser_pwd="$PROJECT_ROOT"/interfacer/ + cp -a "$source_file" "$goreleaser_pwd" + ls -alh "$goreleaser_pwd" +} + +# It is possible to use unsigned webextensions in Firefox but it requires that Firefox +# uses problematically insecure config. I know it's a hassle having to jump through all +# these signing hoops, but I think it's better to use a standard Firefox configuration. +# Moving away from the webextension alltogether is another story, but something I'm still +# thinking about. +# +# NB: There can only be one canonical XPI for each semantic version. +# +# shellcheck disable=2120 +function webextension_sign() { + local use_existing=$1 + if [ "$use_existing" == "" ]; then "$NODE_BIN"/web-ext sign --api-key "$MDN_USER" --api-secret "$MDN_KEY" + _rename_built_xpi else - # TODO: This doesn't currently work with the Marionettte `tempAddon` - # installation method. Just use `web-ext run` and Browsh's `use-existing-ff` - # flag - which is better anyway as it auto-reloads the extension when files - # change. NB: If you fix this, don't forget to change the filename loaded - # by `Asset()` in `main.go`. - # In development/testing, we want to be able to bundle the webextension - # frequently without having to resort to version bumps. - source_file="$source_dir/browsh-$version.zip" + echo "Skipping signing, downloading existing webextension" + local base="https://github.com/browsh-org/browsh/releases/download" + curl -L \ + -o "$(versioned_xpi_file)" \ + "$base/v$LATEST_TAGGED_VERSION/browsh-$LATEST_TAGGED_VERSION.xpi" fi +} - echo "Bundling $source_file to $bundle_file" - cp -f "$source_file" "$bundle_file" +function _rename_built_xpi() { + pushd "$XPI_SOURCE_DIR" || _panic + local xpi_file + xpi_file="$( + find ./*.xpi \ + -printf "%T@ %f\n" | + sort | + cut -d' ' -f2 | + tail -n1 + )" + cp -a "$xpi_file" "$(versioned_xpi_file)" + popd || _panic } function bundle_production_webextension() { local version && version=$(browsh_version) local base='https://github.com/browsh-org/browsh/releases/download' local release_url="$base/v$version/browsh-$version-an.fx.xpi" - local xpi_file=$PROJECT_ROOT/interfacer/src/browsh/browsh.xpi - curl -L -o "$xpi_file" "$release_url" + echo "Downloading webextension from: $release_url" + local size && size=$(wc -c <"$XPI_PATH") + curl -L -o "$XPI_PATH" "$release_url" + if [ "$size" -lt 500 ]; then + _panic "Problem downloading latest webextension XPI" + fi } diff --git a/scripts/common.bash b/scripts/common.bash index 20bcb04..cfa04c9 100644 --- a/scripts/common.bash +++ b/scripts/common.bash @@ -1,7 +1,23 @@ -#!/bin/env bash +#!/usr/bin/env bash +# shellcheck disable=2120 function _panic() { local message=$1 echo >&2 "$message" exit 1 } + +function _md5() { + local path=$1 + md5sum "$path" | cut -d' ' -f1 +} + +function pushd() { + # shellcheck disable=2119 + command pushd "$@" >/dev/null || _panic +} + +function popd() { + # shellcheck disable=2119 + command popd "$@" >/dev/null || _panic +} diff --git a/scripts/docker.bash b/scripts/docker.bash index 8d758a1..e87086f 100644 --- a/scripts/docker.bash +++ b/scripts/docker.bash @@ -1,6 +1,7 @@ #!/usr/bin/env bash function docker_image_name() { + _export_versions echo browsh/browsh:v"$BROWSH_VERSION" } diff --git a/scripts/misc.bash b/scripts/misc.bash index 7932329..e3bd51a 100644 --- a/scripts/misc.bash +++ b/scripts/misc.bash @@ -36,15 +36,19 @@ function install_firefox() { popd || _panic } -function parse_golang_version_from_ci_config() { - local line && line=$(grep 'go-version:' <"$PROJECT_ROOT"/.github/workflows/main.yml) - local version && version=$(echo "$line" | tr -s ' ' | cut -d ' ' -f 3) - [ "$version" = "" ] && _panic "Couldn't parse Golang version" +function parse_golang_version_from_go_mod() { + local path=$1 + [ "$path" = "" ] && _panic "Path to Golang interfacer code not passed" + local line && line=$(grep '^go ' <"$path"/go.mod) + local version && version=$(echo "$line" | tr -s ' ' | cut -d ' ' -f 2) + [ "$(echo "$version" | tr -s ' ')" == "" ] && _panic "Couldn't parse Golang version" echo -n "$version" } function install_golang() { - local version && version=$(parse_golang_version_from_ci_config) + local path=$1 + [ "$path" = "" ] && _panic "Path to Golang interfacer code not passed" + local version && version=$(parse_golang_version_from_go_mod "$path") [ "$GOPATH" = "" ] && _panic "GOPATH not set" [ "$GOROOT" = "" ] && _panic "GOROOT not set" echo "Installing Golang v$version... to $GOROOT" @@ -56,14 +60,3 @@ function install_golang() { tar -C "$GOROOT/.." -xzf go.tar.gz go version } - -function build_browsh_binary() { - local path=$1 - pushd "$path" || _panic - local webextension="src/browsh/browsh.xpi" - [ ! -f "$webextension" ] && _panic "browsh.xpi not present" - md5sum "$webextension" - go build ./cmd/browsh - ./browsh --version - popd || _panic -} diff --git a/scripts/releasing.bash b/scripts/releasing.bash index b474a98..f162fcf 100644 --- a/scripts/releasing.bash +++ b/scripts/releasing.bash @@ -1,4 +1,4 @@ -#!/bin/env bash +#!/usr/bin/env bash export BROWSH_VERSION export LATEST_TAGGED_VERSION @@ -8,8 +8,12 @@ function _goreleaser_production() { echo "Installing \`goreleaser'..." go install github.com/goreleaser/goreleaser@v"$GORELEASER_VERSION" fi - pushd "$PROJECT_ROOT"/interfacer/src || _panic - goreleaser release + pushd "$PROJECT_ROOT"/interfacer || _panic + _export_versions + [ "$BROWSH_VERSION" = "" ] && _panic "BROWSH_VERSION unset (goreleaser needs it)" + goreleaser release \ + --config "$PROJECT_ROOT"/goreleaser.yml \ + --rm-dist popd || _panic } @@ -21,9 +25,9 @@ function _export_versions() { } function _parse_browsh_version() { - version_file=$PROJECT_ROOT/interfacer/src/browsh/version.go - line=$(grep 'browshVersion' <"$version_file") - version=$(echo "$line" | grep -o '".*"' | sed 's/"//g') + local version_file=$PROJECT_ROOT/interfacer/src/browsh/version.go + local line && line=$(grep 'browshVersion' <"$version_file") + local version && version=$(echo "$line" | grep -o '".*"' | sed 's/"//g') echo -n "$version" } @@ -39,7 +43,7 @@ function _tag_on_version_change() { echo_versions if ! _is_new_version; then - echo "Not running release as there's no new version." + echo "Not tagging as there's no new version." exit 0 fi @@ -47,15 +51,13 @@ function _tag_on_version_change() { git show v"$BROWSH_VERSION" --quiet git config --global user.email "ci@github.com" git config --global user.name "Github Actions" - # `/dev/null` needed to prevent Github token appearing in logs - git push --tags --quiet https://"$GITHUB_TOKEN"@github.com/browsh-org/browsh >/dev/null 2>&1 - + git add --all git reset --hard v"$BROWSH_VERSION" } function echo_versions() { _export_versions - echo "Browsh Golang version: $BROWSH_VERSION" + echo "Browsh binary version: $BROWSH_VERSION" echo "Git latest tag: $LATEST_TAGGED_VERSION" } @@ -72,21 +74,35 @@ function github_actions_output_version_status() { echo "::set-output name=is_new_version::$status" } -function npm_build_release() { +function webext_build_release() { pushd "$PROJECT_ROOT"/webext || _panic - BROWSH_ENV=RELEASE npm run build_webextension + build_webextension_production popd || _panic } function update_browsh_website_with_new_version() { + _export_versions + local remote="git@github.com:browsh-org/www.brow.sh.git" pushd /tmp || _panic - git clone https://github.com/browsh-org/www.brow.sh.git - cd www.brow.sh || exit 1 + git clone "$remote" + cd www.brow.sh || _panic echo "latest_version: $BROWSH_VERSION" >_data/browsh.yml git add _data/browsh.yml git commit -m "Github Actions: updated Browsh version to $BROWSH_VERSION" - # `/dev/null` needed to prevent Github token appearing in logs - git push --quiet https://"$GITHUB_TOKEN"@github.com/browsh-org/www.brow.sh >/dev/null 2>&1 + git push "$remote" + popd || _panic +} + +function update_homebrew_tap_with_new_version() { + _export_versions + local remote="git@github.com:browsh-org/homebrew-browsh.git" + pushd /tmp || _panic + git clone "$remote" + cd homebrew-browsh || _panic + cp -f "$PROJECT_ROOT"/interfacer/dist/browsh.rb browsh.rb + git add browsh.rb + git commit -m "Github Actions: updated to $BROWSH_VERSION" + git push "$remote" popd || _panic } @@ -99,9 +115,23 @@ function goreleaser_local_only() { popd || _panic } +function build_browsh_binary() { + # Requires $path argument because it's used in the Dockerfile where the GOROOT is + # outside .git/ + local path=$1 + pushd "$path" || _panic + local webextension="src/browsh/browsh.xpi" + [ ! -f "$webextension" ] && _panic "browsh.xpi not present" + md5sum "$webextension" + go build ./cmd/browsh + echo "Freshly built \`browsh' version: $(./browsh --version 2>&1)" + popd || _panic +} + function release() { - npm_build_release + [ "$(git rev-parse --abbrev-ref HEAD)" != "master" ] && _panic "Not releasing unless on the master branch" + webext_build_release + build_browsh_binary "$PROJECT_ROOT"/interfacer _tag_on_version_change _goreleaser_production - update_browsh_website_with_new_version } diff --git a/webext/package.json b/webext/package.json index 3f2685e..84f9b1a 100644 --- a/webext/package.json +++ b/webext/package.json @@ -1,7 +1,8 @@ { "type": "module", "scripts": { - "build_webextension": "../ctl.sh build_webextension", + "build:dev": "webpack", + "build:watch": "webpack --watch", "lint": "prettier --list-different '{src,test}/**/*.js'", "test": "NODE_PATH=src:test mocha" },