From 545e4214237abfb9082fbff1e1a32f42df030946 Mon Sep 17 00:00:00 2001 From: Orange Mug Date: Fri, 12 May 2023 16:25:14 +0100 Subject: [PATCH] Adds CI for webdriver tests (#1343) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Github action CI workflows added for webdriver tests. I've also refactored the `./scripts/e2e-*` scripts. These scripts were somewhat unique compared to the other scripts. They are now more inline with the other scripts in that directory and run via ``` % yarn e2e --help Usage: yarn e2e [options] Commands: yarn e2e serve start test server yarn e2e test:ci [env] runner for CI (github-actions) yarn e2e test:local run webdriver tests locally yarn e2e test:browserstack run webdriver tests on browserstack yarn e2e selenium:grid start selenium grid (test linux) Options: --help Show help [boolean] --version Show version number [boolean] ``` I've also added an experimental linux runner see https://github.com/tldraw/tldraw/blob/2cca4ddb77ae18f49e9738330ec13d594699570a/e2e/README.md?plain=1#L320-L333 ### Change Type - [x] `tests` — Changes to any testing-related code only (will not publish a new version) ### Release Notes - Github action CI workflows added for webdriver tests - Refactored e2e test runner --- .github/workflows/webdriver-nightly.yml | 65 +++++ .github/workflows/webdriver-on-demand.yml | 118 +++++++++ .github/workflows/webdriver.yml | 67 +++++ apps/webdriver/scripts/dev.mjs | 3 + e2e/README.md | 53 +++- e2e/package.json | 4 +- e2e/wdio.browserstack.conf.js | 282 ++++++++++++++++++++++ e2e/wdio.local.conf.js | 47 ++-- e2e/wdio.nightly.conf.js | 13 +- e2e/wdio.remote.conf.js | 271 --------------------- e2e/wdio.util.js | 65 +++++ package.json | 7 +- public-yarn.lock | 238 ++++++++++++++++-- scripts/e2e-run-ci | 14 -- scripts/e2e-run-tests | 9 - scripts/e2e-start-server | 3 - scripts/e2e/commands/index.ts | 7 + scripts/e2e/commands/selenium-grid.ts | 36 +++ scripts/e2e/commands/serve.ts | 8 + scripts/e2e/commands/test-browserstack.ts | 20 ++ scripts/e2e/commands/test-ci.ts | 60 +++++ scripts/e2e/commands/test-local.ts | 14 ++ scripts/e2e/commands/util.ts | 16 ++ scripts/e2e/index.ts | 94 ++++++++ 24 files changed, 1152 insertions(+), 362 deletions(-) create mode 100644 .github/workflows/webdriver-nightly.yml create mode 100644 .github/workflows/webdriver-on-demand.yml create mode 100644 .github/workflows/webdriver.yml create mode 100644 e2e/wdio.browserstack.conf.js delete mode 100644 e2e/wdio.remote.conf.js create mode 100644 e2e/wdio.util.js delete mode 100755 scripts/e2e-run-ci delete mode 100755 scripts/e2e-run-tests delete mode 100755 scripts/e2e-start-server create mode 100644 scripts/e2e/commands/index.ts create mode 100644 scripts/e2e/commands/selenium-grid.ts create mode 100644 scripts/e2e/commands/serve.ts create mode 100644 scripts/e2e/commands/test-browserstack.ts create mode 100644 scripts/e2e/commands/test-ci.ts create mode 100644 scripts/e2e/commands/test-local.ts create mode 100644 scripts/e2e/commands/util.ts create mode 100644 scripts/e2e/index.ts diff --git a/.github/workflows/webdriver-nightly.yml b/.github/workflows/webdriver-nightly.yml new file mode 100644 index 000000000..26f2dc956 --- /dev/null +++ b/.github/workflows/webdriver-nightly.yml @@ -0,0 +1,65 @@ +name: Webdriver nightly (browserstack) + +on: + workflow_dispatch: + schedule: + - cron: '0 2 * * *' # run at 2 AM UTC + +jobs: + test: + name: 'nightly' + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest-16-cores-open] + node-version: [16] + + container: + image: node:${{ matrix.node-version }} + options: --network-alias testhost + volumes: + - /home/runner/work/_temp/e2e:/home/runner/work/_temp/e2e + + steps: + # start browserstack + - name: 'BrowserStack Env Setup' # Invokes the setup-env action + uses: browserstack/github-actions/setup-env@master + with: + username: jamieblair_YXsTBS + access-key: BUcyZn9PF4iwKgayXinm + - name: 'BrowserStack Local Tunnel Setup' # Invokes the setup-local action + uses: browserstack/github-actions/setup-local@master + with: + local-testing: start + local-identifier: random + + - name: Check out code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: true + + - name: Setup Node.js environment + uses: actions/setup-node@v3 + with: + node-version: 18 + cache: 'yarn' + cache-dependency-path: 'public-yarn.lock' + + - name: Enable corepack + run: corepack enable + + - name: Install dependencies + run: yarn + + - name: Run server + run: yarn dev-webdriver & + + - run: yarn e2e test:ci nightly + env: + CI: true + DOWNLOADS_DIR: '/home/runner/work/_temp/e2e/' + TEST_URL: 'https://testhost:5421' + WB_BUILD_NAME: 'nightly' diff --git a/.github/workflows/webdriver-on-demand.yml b/.github/workflows/webdriver-on-demand.yml new file mode 100644 index 000000000..16f0b6baa --- /dev/null +++ b/.github/workflows/webdriver-on-demand.yml @@ -0,0 +1,118 @@ +name: Webdriver on demand (browserstack) + +on: + workflow_dispatch: + inputs: + WD_BROWSER_CHROME: + description: 'Chrome' + required: false + default: true + type: boolean + WD_BROWSER_FIREFOX: + description: 'Firefox' + required: false + default: true + type: boolean + WD_BROWSER_EDGE: + description: 'Edge' + required: false + default: true + type: boolean + WD_BROWSER_SAFARI: + description: 'Safari' + required: false + default: true + type: boolean + WD_BROWSER_SAMSUNG: + description: 'Samsung' + required: false + default: true + type: boolean + WD_OS_WINDOWS: + description: 'Windows' + required: false + default: true + type: boolean + WD_OS_MACOS: + description: 'MacOS' + required: false + default: true + type: boolean + WD_OS_ANDROID: + description: 'Android' + required: false + default: true + type: boolean + WD_OS_IOS: + description: 'iOS' + required: false + default: true + type: boolean + +jobs: + test: + name: 'on-demand' + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest-16-cores-open] + node-version: [16] + + container: + image: node:${{ matrix.node-version }} + options: --network-alias testhost + volumes: + - /home/runner/work/_temp/e2e:/home/runner/work/_temp/e2e + + steps: + # start browserstack + - name: 'BrowserStack Env Setup' # Invokes the setup-env action + uses: browserstack/github-actions/setup-env@master + with: + username: jamieblair_YXsTBS + access-key: BUcyZn9PF4iwKgayXinm + - name: 'BrowserStack Local Tunnel Setup' # Invokes the setup-local action + uses: browserstack/github-actions/setup-local@master + with: + local-testing: start + local-identifier: random + + - name: Check out code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: true + + - name: Setup Node.js environment + uses: actions/setup-node@v3 + with: + node-version: 18 + cache: 'yarn' + cache-dependency-path: 'public-yarn.lock' + + - name: Enable corepack + run: corepack enable + + - name: Install dependencies + run: yarn + + - name: Run server + run: yarn dev-webdriver & + + - run: yarn e2e test:ci nightly + env: + CI: true + DOWNLOADS_DIR: '/home/runner/work/_temp/e2e/' + TEST_URL: 'https://testhost:5421' + WD_BROWSER_CHROME: ${{ inputs.WD_BROWSER_CHROME }} + WD_BROWSER_FIREFOX: ${{ inputs.WD_BROWSER_FIREFOX }} + WD_BROWSER_EDGE: ${{ inputs.WD_BROWSER_EDGE }} + WD_BROWSER_SAFARI: ${{ inputs.WD_BROWSER_SAFARI }} + WD_BROWSER_SAMSUNG: ${{ inputs.WD_BROWSER_SAMSUNG }} + WD_OS_WINDOWS: ${{ inputs.WD_OS_WINDOWS }} + WD_OS_MACOS: ${{ inputs.WD_OS_MACOS }} + WD_OS_ANDROID: ${{ inputs.WD_OS_ANDROID }} + WD_OS_IOS: ${{ inputs.WD_OS_IOS }} + WB_BUILD_NAME: 'ondemand' diff --git a/.github/workflows/webdriver.yml b/.github/workflows/webdriver.yml new file mode 100644 index 000000000..8c12bc138 --- /dev/null +++ b/.github/workflows/webdriver.yml @@ -0,0 +1,67 @@ +name: Webdriver checks + +on: + merge_group: + pull_request: + branches: [main, production] + push: + branches: [main, production] + +jobs: + test: + name: 'test/standalone-${{ matrix.browser }} (${{ matrix.os }})' + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest-16-cores-open] + node-version: [16] + browser: [chrome] + browser-version: ['111.0'] + + container: + image: node:${{ matrix.node-version }} + options: --network-alias testhost + volumes: + - /home/runner/work/_temp/e2e:/home/runner/work/_temp/e2e + + services: + selenium: + image: selenium/standalone-${{ matrix.browser }}:${{ matrix.browser-version }} + ports: + - 4444:4444 + options: --shm-size=2gb + volumes: + - /home/runner/work/_temp/e2e/:/home/seluser/files + + steps: + - name: Check out code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: true + + - name: Setup Node.js environment + uses: actions/setup-node@v3 + with: + node-version: 18 + cache: 'yarn' + cache-dependency-path: 'public-yarn.lock' + + - name: Enable corepack + run: corepack enable + + - name: Install dependencies + run: yarn + + - run: DOCKER_HOST=selenium yarn e2e test:ci local + env: + CI: true + DOWNLOADS_DIR: /home/runner/work/_temp/e2e/ + BROWSER: ${{ matrix.browser }} + TEST_URL: "https://testhost:5421" + GH_EVENT_NAME: ${{ github.event_name }} + GH_PR_NUMBER: ${{ github.pull_request.number }} + + diff --git a/apps/webdriver/scripts/dev.mjs b/apps/webdriver/scripts/dev.mjs index bdaec49cf..6d446b173 100644 --- a/apps/webdriver/scripts/dev.mjs +++ b/apps/webdriver/scripts/dev.mjs @@ -187,6 +187,9 @@ async function main() { console.log(e) }) sslServer.listen(SSL_PORT, () => { + // TODO: Nasty, but gets detected by script runner + log('[tldraw:process_ready]'); + log(`Running on:\n`) log(chalk.bold().cyan(` https://localhost:${SSL_PORT}`)) log(`\nNetwork:\n`) diff --git a/e2e/README.md b/e2e/README.md index f60a632f2..b9a4c3217 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -6,31 +6,50 @@ Webdriver testing can be tricky because you're sending commands to an actual bro > **A note on stability**: Webdriver tests are a lot more flakey than other types of testing, the major benefit is that you can run them on real devices, so we can hopefully get a good smoke test of various real devices. You can also screenshot those devices during test runs, to check look. You however probably don't want to write too many webdriver tests, they are best placed for smoke testing and testing stuff that's very browser specific. +There is a script called `yarn e2e`, running `yarn e2e --help` will show you it's usage + +``` +Usage: yarn e2e [options] + +Commands: + yarn e2e serve start test server + yarn e2e test:ci runner for CI (github-actions) + yarn e2e test:local run webdriver tests locally + yarn e2e test:browserstack run webdriver tests on browserstack + yarn e2e selenium:grid start selenium grid (test linux) + +Options: + --help Show help [boolean] + --version Show version number [boolean] +``` + To run the tests you first must start the server with ```sh -./scripts/e2e-start-server +yarn e2e serve ``` You can then either test locally with ```sh -./scripts/e2e-run-tests local +yarn e2e test:local ``` By default it'll just run the chrome tests. You can also specify other browsers to run like this ```sh # Note edge, safari are a work in progress and will just be skipped for now. -./scripts/e2e-run-tests local firefox,chrome,edge,safari +yarn e2e test:local -b firefox,chrome,edge,safari ``` Or to test remotely via browserstack ```sh -./scripts/e2e-run-tests remote +yarn e2e test:browserstack ``` +**Note**: You'll need to set `BROWSERSTACK_USER` and `BROWSERSTACK_KEY` in your environment, which you can grab from + There are three parts to the testing API - `runtime` — the tldraw `App` instance in the browser window. This calls methods in the JS runtime of the app @@ -139,13 +158,13 @@ To run the above tests, we'd probably want to first isolate the test and `only` Now lets start the dev server and run the tests locally. In one terminal session run ```sh -./scripts/e2e-start-server +yarn e2e serve ``` In another terminal session run ```sh -./scripts/e2e-run-tests local +yarn e2e test:local ``` The test should run twice in chrome. Once in desktop chrome and once chrome mobile emulation mode. The results should look something like @@ -176,15 +195,15 @@ Spec Files: 2 passed, 2 total (100% completed) in 00:00:05 Yay, passing tests 🎉 -However we're not done quite yet. Next up, we need to test them across the rest of our supported browsers. With the `e2e-start-server` still running +However we're not done quite yet. Next up, we need to test them across the rest of our supported browsers. With the test server still running ```sh -./scripts/e2e-run-tests remote +yarn e2e test:browserstack ``` This will start a tunnel from browserstack to your local machine, running the tests against the local server. You can head to browserstack to see the completed test suite -In you terminal you should see the results +In your terminal you should see the results ``` ------------------------------------------------------------------ @@ -296,3 +315,19 @@ This module isn't actually used but is required for `wdio-edgedriver-service` to ### `safaridriver` Locally safari webdriver tests are currently somewhat buggy, there appear to be some tests that don't complete. Please take on this task if you have time 🙂 + + +## Experimental +You can now also run _linux_ tests on _macos_. To do this start up a selenium grid with. Note, you **must** have `docker` installed locally. + +```sh +yarn e2e selenium:grid +``` + +Then run + +```sh +yarn e2e test:local --os linux -b firefox +``` + +**Note**: Currently only `firefox` is supported. You can hit to see a VNC of the app runing in a docker container. \ No newline at end of file diff --git a/e2e/package.json b/e2e/package.json index 27a885db9..430656a69 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -27,7 +27,7 @@ ], "scripts": { "test:local": "TS_NODE_PROJECT=./tsconfig.json wdio run wdio.local.conf.js", - "test:remote": "TS_NODE_PROJECT=./tsconfig.json wdio run wdio.remote.conf.js", + "test:browserstack": "TS_NODE_PROJECT=./tsconfig.json wdio run wdio.browserstack.conf.js", "test:nightly": "TS_NODE_PROJECT=./tsconfig.json wdio run wdio.nightly.conf.js", "lint": "yarn run -T tsx ../scripts/lint.ts" }, @@ -37,7 +37,7 @@ "@tldraw/primitives": "workspace:*", "@types/mocha": "^10.0.1", "@types/sharp": "^0.31.1", - "@wdio/browserstack-service": "^8.1.3", + "@wdio/browserstack-service": "^8.10.1", "@wdio/cli": "^8.1.3", "@wdio/globals": "^8.1.3", "@wdio/local-runner": "^8.1.2", diff --git a/e2e/wdio.browserstack.conf.js b/e2e/wdio.browserstack.conf.js new file mode 100644 index 000000000..0c95ab511 --- /dev/null +++ b/e2e/wdio.browserstack.conf.js @@ -0,0 +1,282 @@ +const { BUILD_NAME, logBrowserstackUrl, filterCapabilities } = require('./wdio.util') + +global.webdriverService = 'browserstack' +global.webdriverTestUrl = 'http://localhost:5420/' + +const capabilities = [ + /** + * ==================================================================== + * Windows 11 + * ==================================================================== + */ + { + 'bstack:options': { + os: 'Windows', + osVersion: '11', + browserVersion: 'latest', + seleniumVersion: '3.14.0', + }, + browserName: 'Chrome', + 'tldraw:options': { + browser: 'chrome', + os: 'win32', + ui: 'desktop', + device: 'desktop', + input: ['mouse'], + }, + }, + { + 'bstack:options': { + os: 'Windows', + osVersion: '11', + browserVersion: 'latest', + seleniumVersion: '4.6.0', + }, + acceptInsecureCerts: 'true', + browserName: 'Edge', + 'tldraw:options': { + browser: 'edge', + os: 'win32', + ui: 'desktop', + device: 'desktop', + input: ['mouse'], + }, + }, + { + 'bstack:options': { + os: 'Windows', + osVersion: '11', + browserVersion: 'latest', + seleniumVersion: '4.6.0', + }, + acceptInsecureCerts: 'true', + browserName: 'Firefox', + 'tldraw:options': { + browser: 'firefox', + os: 'win32', + ui: 'desktop', + device: 'desktop', + input: ['mouse'], + }, + }, + /** + * ==================================================================== + * MacOS + * ==================================================================== + */ + // { + // 'bstack:options' : { + // "os" : "OS X", + // "osVersion" : "Ventura", + // "browserVersion" : "16.0", + // "seleniumVersion" : "4.6.0", + // }, + // "acceptInsecureCerts" : "true", + // "browserName" : "Safari", + // }, + { + 'bstack:options': { + os: 'OS X', + osVersion: 'Ventura', + browserVersion: 'latest', + seleniumVersion: '4.6.0', + }, + browserName: 'Chrome', + 'tldraw:options': { + browser: 'chrome', + os: 'darwin', + ui: 'desktop', + device: 'desktop', + input: ['mouse'], + }, + }, + { + 'bstack:options': { + os: 'OS X', + osVersion: 'Ventura', + browserVersion: 'latest', + seleniumVersion: '4.6.0', + }, + acceptInsecureCerts: 'true', + browserName: 'Firefox', + 'tldraw:options': { + browser: 'firefox', + os: 'darwin', + ui: 'desktop', + device: 'desktop', + input: ['mouse'], + }, + }, + { + 'bstack:options': { + os: 'OS X', + osVersion: 'Ventura', + browserVersion: 'latest', + seleniumVersion: '4.6.0', + }, + acceptInsecureCerts: 'true', + browserName: 'Edge', + 'tldraw:options': { + browser: 'edge', + os: 'darwin', + ui: 'desktop', + device: 'desktop', + input: ['mouse'], + }, + }, + /** + // * ==================================================================== + // * Android + // * ==================================================================== + // */ + { + 'bstack:options': { + osVersion: '13.0', + deviceName: 'Google Pixel 7', + appiumVersion: '1.22.0', + }, + browserName: 'chrome', + 'tldraw:options': { + appium: true, + browser: 'chrome', + os: 'android', + ui: 'mobile', + device: 'mobile', + input: ['touch'], + }, + }, + { + 'bstack:options': { + osVersion: '11.0', + deviceName: 'Samsung Galaxy S21', + appiumVersion: '1.22.0', + }, + acceptInsecureCerts: 'true', + browserName: 'samsung', + 'tldraw:options': { + appium: true, + browser: 'samsung', + os: 'android', + ui: 'mobile', + device: 'mobile', + input: ['touch'], + }, + }, + { + 'bstack:options': { + osVersion: '11.0', + deviceName: 'Samsung Galaxy S21', + appiumVersion: '1.22.0', + }, + acceptInsecureCerts: 'true', + browserName: 'chrome', + 'tldraw:options': { + appium: true, + browser: 'chrome', + os: 'android', + ui: 'mobile', + device: 'mobile', + input: ['touch'], + }, + }, + /** + * ==================================================================== + * iOS + * ==================================================================== + */ + // { + // 'bstack:options': { + // "osVersion" : "16", + // "deviceName" : "iPhone 14", + // "appiumVersion": "1.22.0" + // }, + // "acceptInsecureCerts" : "true", + // "browserName" : "safari", + // }, + // { + // 'bstack:options': { + // "osVersion" : "16", + // "deviceName" : "iPad Pro 12.9 2022", + // "appiumVersion": "1.22.0" + // }, + // "acceptInsecureCerts" : "true", + // "browserName" : "safari", + // }, +].map((capability) => { + return { + ...capability, + acceptInsecureCerts: true, + 'bstack:options': { + ...capability['bstack:options'], + projectName: 'tldraw', + buildName: BUILD_NAME, + consoleLogs: 'verbose', + }, + 'tldraw:options': { + ...capability['tldraw:options'], + }, + } +}) + +exports.config = { + user: process.env.BROWSERSTACK_USER, + key: process.env.BROWSERSTACK_KEY, + hostname: 'hub.browserstack.com', + specs: ['./test/specs/index.ts'], + services: [ + [ + 'browserstack', + { + browserstackLocal: true, + testObservability: true, + testObservabilityOptions: { + projectName: 'tldraw', + buildName: BUILD_NAME, + buildTag: process.env.GITHUB_SHA || 'local', + }, + opts: { + verbose: 'true', + }, + }, + ], + ], + exclude: [], + maxInstances: 1, + waitforInterval: 200, + /** + * Capabilities can be configured via + * + * The once commented out currently fail on because of insecure certs, details + */ + capabilities: filterCapabilities(capabilities), + bail: 0, + waitforTimeout: 10000, + connectionRetryTimeout: 120000, + connectionRetryCount: 3, + framework: 'mocha', + reporters: ['spec'], + mochaOpts: { + ui: 'bdd', + timeout: 5 * 60 * 1000, + }, + logLevel: process.env.WD_LOG_LEVEL ?? 'info', + coloredLogs: true, + screenshotPath: './errorShots/', + waitforTimeout: 30000, + connectionRetryTimeout: 90000, + connectionRetryCount: 3, + beforeSession: (_config, capabilities) => { + global.tldrawOptions = capabilities['tldraw:options'] + }, + afterSession: async (_config, capabilities, _specs) => { + await logBrowserstackUrl() + }, + autoCompileOpts: { + autoCompile: true, + tsNodeOpts: { + transpileOnly: true, + swc: true, + project: './tsconfig.json', + }, + }, +} diff --git a/e2e/wdio.local.conf.js b/e2e/wdio.local.conf.js index 6a512ac45..36c6c16cc 100644 --- a/e2e/wdio.local.conf.js +++ b/e2e/wdio.local.conf.js @@ -1,10 +1,7 @@ const edgeDriver = require('@sitespeed.io/edgedriver') +const { filterCapabilities } = require('./wdio.util') -const CURRENT_OS = { - win32: 'windows', - linux: 'linux', - darwin: 'macos', -}[process.platform] +const CURRENT_OS = process.platform global.webdriverService = 'local' global.webdriverTestUrl = process.env.TEST_URL ?? 'http://localhost:5420/' @@ -158,32 +155,22 @@ if (process.env.CI === 'true') { input: ['mouse'], }, }, + { + maxInstances: 1, + browserName: 'firefox', + platformName: 'Linux', + acceptInsecureCerts: true, + 'tldraw:options': { + browser: 'firefox', + os: 'linux', + ui: 'desktop', + device: 'desktop', + input: ['mouse'], + }, + }, ] } -let browsers = (process.env.BROWSERS || 'chrome').split(',').map((b) => b.trim()) -const validBrowsers = ['chrome', 'safari', 'firefox', 'edge', 'vscode'] -const skippedBrowsers = [] - -if (browsers.includes('safari')) { - console.log( - 'NOTE: In safari you need to run `safaridriver --enable`, see for details.' - ) -} - -for (const browser of browsers) { - if (!validBrowsers.includes(browser)) { - throw new Error(`'${browser}' not a valid browser name`) - } - if (skippedBrowsers.includes(browser)) { - console.error(`'${browser}' not currently supported`) - } -} - -capabilities = capabilities.filter((capability) => { - return browsers.includes(capability['tldraw:options'].browser) -}) - exports.config = { specs: ['./test/specs/index.ts'], hostname: process.env.DOCKER_HOST || 'localhost', @@ -219,7 +206,7 @@ exports.config = { // HACK: If we don't have edge as a capability but we do have // this service then `wdio-edgedriver-service` throws an scary // error (which doesn't actually effect anything) - ...(!browsers.includes('edge') + ...(!process.env.BROWSERS.split(',').includes('edge') ? [] : [ [ @@ -234,7 +221,7 @@ exports.config = { ]), ], maxInstances: 1, - capabilities: capabilities, + capabilities: filterCapabilities(capabilities), logLevel: process.env.WD_LOG_LEVEL ?? 'error', bail: 0, baseUrl: 'http://localhost', diff --git a/e2e/wdio.nightly.conf.js b/e2e/wdio.nightly.conf.js index 6190b4bff..4f416f151 100644 --- a/e2e/wdio.nightly.conf.js +++ b/e2e/wdio.nightly.conf.js @@ -1,4 +1,4 @@ -const BUILD_NAME = `test-suite-${new Date().toISOString()}` +const { BUILD_NAME, logBrowserstackUrl } = require('./wdio.util') global.webdriverService = 'browserstack' global.webdriverTestUrl = 'http://localhost:5420/' @@ -13,6 +13,12 @@ exports.config = { 'browserstack', { browserstackLocal: true, + testObservability: true, + testObservabilityOptions: { + projectName: 'tldraw', + buildName: BUILD_NAME, + buildTag: process.env.GITHUB_SHA || 'local', + }, opts: { verbose: 'true', }, @@ -233,7 +239,7 @@ exports.config = { acceptInsecureCerts: true, 'bstack:options': { ...capability['bstack:options'], - projectName: 'smoke-tests', + projectName: 'tldraw', buildName: BUILD_NAME, consoleLogs: 'verbose', }, @@ -269,6 +275,9 @@ exports.config = { beforeSession: (_config, capabilities) => { global.tldrawOptions = capabilities['tldraw:options'] }, + afterSession: async (_config, capabilities, _specs) => { + await logBrowserstackUrl() + }, autoCompileOpts: { autoCompile: true, tsNodeOpts: { diff --git a/e2e/wdio.remote.conf.js b/e2e/wdio.remote.conf.js deleted file mode 100644 index 133d3addd..000000000 --- a/e2e/wdio.remote.conf.js +++ /dev/null @@ -1,271 +0,0 @@ -const BUILD_NAME = `test-suite-${new Date().toISOString()}` - -global.webdriverService = 'browserstack' -global.webdriverTestUrl = 'http://localhost:5420/' - -exports.config = { - user: process.env.BROWSERSTACK_USER, - key: process.env.BROWSERSTACK_KEY, - hostname: 'hub.browserstack.com', - specs: ['./test/specs/index.ts'], - services: [ - [ - 'browserstack', - { - browserstackLocal: true, - opts: { - verbose: 'true', - }, - }, - ], - ], - exclude: [], - maxInstances: 1, - waitforInterval: 200, - /** - * Capabilities can be configured via - * - * The once commented out currently fail on because of insecure certs, details - */ - capabilities: [ - /** - * ==================================================================== - * Windows 11 - * ==================================================================== - */ - { - 'bstack:options': { - os: 'Windows', - osVersion: '11', - browserVersion: 'latest', - seleniumVersion: '3.14.0', - }, - browserName: 'Chrome', - 'tldraw:options': { - browser: 'chrome', - os: 'windows', - ui: 'desktop', - device: 'desktop', - input: ['mouse'], - }, - }, - { - 'bstack:options': { - os: 'Windows', - osVersion: '11', - browserVersion: 'latest', - seleniumVersion: '4.6.0', - }, - acceptInsecureCerts: 'true', - browserName: 'Edge', - 'tldraw:options': { - browser: 'edge', - os: 'windows', - ui: 'desktop', - device: 'desktop', - input: ['mouse'], - }, - }, - { - 'bstack:options': { - os: 'Windows', - osVersion: '11', - browserVersion: 'latest', - seleniumVersion: '4.6.0', - }, - acceptInsecureCerts: 'true', - browserName: 'Firefox', - 'tldraw:options': { - browser: 'firefox', - os: 'windows', - ui: 'desktop', - device: 'desktop', - input: ['mouse'], - }, - }, - /** - * ==================================================================== - * MacOS - * ==================================================================== - */ - // { - // 'bstack:options' : { - // "os" : "OS X", - // "osVersion" : "Ventura", - // "browserVersion" : "16.0", - // "seleniumVersion" : "4.6.0", - // }, - // "acceptInsecureCerts" : "true", - // "browserName" : "Safari", - // }, - { - 'bstack:options': { - os: 'OS X', - osVersion: 'Ventura', - browserVersion: 'latest', - seleniumVersion: '4.6.0', - }, - browserName: 'Chrome', - 'tldraw:options': { - browser: 'chrome', - os: 'macos', - ui: 'desktop', - device: 'desktop', - input: ['mouse'], - }, - }, - { - 'bstack:options': { - os: 'OS X', - osVersion: 'Ventura', - browserVersion: 'latest', - seleniumVersion: '4.6.0', - }, - acceptInsecureCerts: 'true', - browserName: 'Firefox', - 'tldraw:options': { - browser: 'firefox', - os: 'macos', - ui: 'desktop', - device: 'desktop', - input: ['mouse'], - }, - }, - { - 'bstack:options': { - os: 'OS X', - osVersion: 'Ventura', - browserVersion: 'latest', - seleniumVersion: '4.6.0', - }, - acceptInsecureCerts: 'true', - browserName: 'Edge', - 'tldraw:options': { - browser: 'edge', - os: 'macos', - ui: 'desktop', - device: 'desktop', - input: ['mouse'], - }, - }, - /** - // * ==================================================================== - // * Android - // * ==================================================================== - // */ - { - 'bstack:options': { - osVersion: '13.0', - deviceName: 'Google Pixel 7', - appiumVersion: '1.22.0', - }, - browserName: 'chrome', - 'tldraw:options': { - appium: true, - browser: 'chrome', - os: 'android', - ui: 'mobile', - device: 'mobile', - input: ['touch'], - }, - }, - { - 'bstack:options': { - osVersion: '11.0', - deviceName: 'Samsung Galaxy S21', - appiumVersion: '1.22.0', - }, - acceptInsecureCerts: 'true', - browserName: 'samsung', - 'tldraw:options': { - appium: true, - browser: 'samsung', - os: 'android', - ui: 'mobile', - device: 'mobile', - input: ['touch'], - }, - }, - { - 'bstack:options': { - osVersion: '11.0', - deviceName: 'Samsung Galaxy S21', - appiumVersion: '1.22.0', - }, - acceptInsecureCerts: 'true', - browserName: 'chrome', - 'tldraw:options': { - appium: true, - browser: 'chrome', - os: 'android', - ui: 'mobile', - device: 'mobile', - input: ['touch'], - }, - }, - /** - * ==================================================================== - * iOS - * ==================================================================== - */ - // { - // 'bstack:options': { - // "osVersion" : "16", - // "deviceName" : "iPhone 14", - // "appiumVersion": "1.22.0" - // }, - // "acceptInsecureCerts" : "true", - // "browserName" : "safari", - // }, - // { - // 'bstack:options': { - // "osVersion" : "16", - // "deviceName" : "iPad Pro 12.9 2022", - // "appiumVersion": "1.22.0" - // }, - // "acceptInsecureCerts" : "true", - // "browserName" : "safari", - // }, - ].map((capability) => { - return { - ...capability, - acceptInsecureCerts: true, - 'bstack:options': { - ...capability['bstack:options'], - projectName: 'smoke-tests', - buildName: BUILD_NAME, - consoleLogs: 'verbose', - }, - 'tldraw:options': { - ...capability['tldraw:options'], - }, - } - }), - bail: 0, - waitforTimeout: 10000, - connectionRetryTimeout: 120000, - connectionRetryCount: 3, - framework: 'mocha', - reporters: ['spec'], - mochaOpts: { - ui: 'bdd', - timeout: 5 * 60 * 1000, - }, - logLevel: process.env.WD_LOG_LEVEL ?? 'info', - coloredLogs: true, - screenshotPath: './errorShots/', - waitforTimeout: 30000, - connectionRetryTimeout: 90000, - connectionRetryCount: 3, - beforeSession: (_config, capabilities) => { - global.tldrawOptions = capabilities['tldraw:options'] - }, - autoCompileOpts: { - autoCompile: true, - tsNodeOpts: { - transpileOnly: true, - swc: true, - project: './tsconfig.json', - }, - }, -} diff --git a/e2e/wdio.util.js b/e2e/wdio.util.js new file mode 100644 index 000000000..6a489e435 --- /dev/null +++ b/e2e/wdio.util.js @@ -0,0 +1,65 @@ +let BUILD_NAME = 'e2e' +if (process.env.GH_EVENT_NAME === 'pull_request') { + BUILD_NAME += `-pr-${process.env.GH_PR_NUMBER}` +} else if (process.env.WB_BUILD_NAME) { + BUILD_NAME += `-${process.env.WB_BUILD_NAME}` +} + +async function logBrowserstackUrl() { + const sessionId = capabilities['webdriver.remote.sessionid'] + + const headers = new Headers() + headers.set( + 'Authorization', + 'Basic ' + btoa(process.env.BROWSERSTACK_USER + ':' + process.env.BROWSERSTACK_KEY) + ) + + const resp = await fetch(`https://api.browserstack.com/automate/sessions/${sessionId}.json`, { + method: 'GET', + headers: headers, + }) + + const respJson = await resp.json() + console.log(`================================== +browser_url: <${respJson.automation_session.browser_url}> +==================================`) +} + +function filterCapabilities(capabilities) { + let browsers = (process.env.BROWSERS || 'chrome').split(',').map((b) => b.trim()) + const validBrowsers = ['chrome', 'safari', 'firefox', 'edge', 'vscode'] + const skippedBrowsers = [] + + if (browsers.includes('safari')) { + console.log( + 'NOTE: In safari you need to run `safaridriver --enable`, see for details.' + ) + } + + for (const browser of browsers) { + if (!validBrowsers.includes(browser)) { + throw new Error(`'${browser}' not a valid browser name`) + } + if (skippedBrowsers.includes(browser)) { + console.error(`'${browser}' not currently supported`) + } + } + + // let oses = (process.env.OS || process.platform).split(',').map((b) => b.trim()) + // const validOses = ['darwin', 'win32', 'linux'] + + // for (const os of oses) { + // if (!validOses.includes(os)) { + // throw new Error(`'${os}' not a valid OS name`) + // } + // } + + const filterFn = (capability) => { + return browsers.includes(capability['tldraw:options'].browser) + // oses.includes(capability['tldraw:options'].os) + } + + return capabilities.filter(filterFn) +} + +module.exports = { BUILD_NAME, logBrowserstackUrl, filterCapabilities } diff --git a/package.json b/package.json index 42da40025..30c05d0d4 100644 --- a/package.json +++ b/package.json @@ -50,7 +50,8 @@ "typecheck": "yarn refresh-assets && tsx scripts/typecheck.ts", "check-scripts": "tsx scripts/check-scripts.ts", "api-check": "lazy api-check", - "test": "lazy test" + "test": "lazy test", + "e2e": "tsx scripts/e2e/index.ts" }, "engines": { "npm": ">=7.0.0" @@ -96,8 +97,10 @@ "json5": "^2.2.3", "lazyrepo": "0.0.0-alpha.26", "rimraf": "^4.4.0", + "tree-kill": "^1.2.2", "tsx": "^3.12.2", - "vercel": "^28.16.15" + "vercel": "^28.16.15", + "yargs": "^17.7.2" }, "resolutions": { "@microsoft/api-extractor@^7.34.1": "patch:@microsoft/api-extractor@npm%3A7.34.1#./.yarn/patches/@microsoft-api-extractor-npm-7.34.1-af268a32f8.patch" diff --git a/public-yarn.lock b/public-yarn.lock index 62eb94c5b..d22a63954 100644 --- a/public-yarn.lock +++ b/public-yarn.lock @@ -3418,6 +3418,30 @@ __metadata: languageName: node linkType: hard +"@puppeteer/browsers@npm:1.0.1": + version: 1.0.1 + resolution: "@puppeteer/browsers@npm:1.0.1" + dependencies: + debug: 4.3.4 + extract-zip: 2.0.1 + http-proxy-agent: 5.0.0 + https-proxy-agent: 5.0.1 + progress: 2.0.3 + proxy-from-env: 1.1.0 + tar-fs: 2.1.1 + unbzip2-stream: 1.4.3 + yargs: 17.7.1 + peerDependencies: + typescript: ">= 4.7.4" + peerDependenciesMeta: + typescript: + optional: true + bin: + browsers: lib/cjs/main-cli.js + checksum: 0dc83d75b0799b8bcc077a17bff3f01d24e4bc3b4449a9fa3f30247eca3b9deb3e7a2fe2f31efb32ed75ca23bd0137ad28b86412a0c1fbce9bcd1aa2ed7bfeb0 + languageName: node + linkType: hard + "@radix-ui/number@npm:1.0.0": version: 1.0.0 resolution: "@radix-ui/number@npm:1.0.0" @@ -4556,7 +4580,7 @@ __metadata: "@tldraw/primitives": "workspace:*" "@types/mocha": ^10.0.1 "@types/sharp": ^0.31.1 - "@wdio/browserstack-service": ^8.1.3 + "@wdio/browserstack-service": ^8.10.1 "@wdio/cli": ^8.1.3 "@wdio/globals": ^8.1.3 "@wdio/local-runner": ^8.1.2 @@ -4670,9 +4694,11 @@ __metadata: prettier: ^2.8.6 prettier-plugin-organize-imports: ^3.2.2 rimraf: ^4.4.0 + tree-kill: ^1.2.2 tsx: ^3.12.2 typescript: ^5.0.2 vercel: ^28.16.15 + yargs: ^17.7.2 languageName: unknown linkType: soft @@ -5428,6 +5454,13 @@ __metadata: languageName: node linkType: hard +"@types/node@npm:^20.1.0": + version: 20.1.1 + resolution: "@types/node@npm:20.1.1" + checksum: 47961ee23f873c14c3f6045422ff3059f3bfb10231ef3080a7a72d7215cc8c2623fa8cedb7b246305962fa9c1e0c9e381e04b12eb3e9ec5d076025c6231ac8da + languageName: node + linkType: hard + "@types/normalize-package-data@npm:^2.4.1": version: 2.4.1 resolution: "@types/normalize-package-data@npm:2.4.1" @@ -6095,24 +6128,24 @@ __metadata: languageName: node linkType: hard -"@wdio/browserstack-service@npm:^8.1.3": - version: 8.10.0 - resolution: "@wdio/browserstack-service@npm:8.10.0" +"@wdio/browserstack-service@npm:^8.10.1": + version: 8.10.1 + resolution: "@wdio/browserstack-service@npm:8.10.1" dependencies: "@types/gitconfiglocal": ^2.0.1 "@wdio/logger": 8.6.6 - "@wdio/reporter": 8.10.0 - "@wdio/types": 8.10.0 + "@wdio/reporter": 8.10.1 + "@wdio/types": 8.10.1 browserstack-local: ^1.5.1 form-data: ^4.0.0 git-repo-info: ^2.1.1 gitconfiglocal: ^2.1.0 got: ^12.1.0 uuid: ^8.3.2 - webdriverio: 8.10.0 + webdriverio: 8.10.1 peerDependencies: "@wdio/cli": ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0 - checksum: 93e19782e68f4780e5fc1e55053f7ba3c9cc968698a8e6130cdfed47cdf12b1bac3147b0edf31d4d763f88b606b92030d617ab296081a0f0880a953752b6aae6 + checksum: 4bf3458e7926f3613ea4743168d5290b67e4b74cad327a4e967f08a3065007d5f4cb73f5e890248c2b4e28d96c7a27f790125527a6be88a7f4efc933be7c0f65 languageName: node linkType: hard @@ -6165,6 +6198,22 @@ __metadata: languageName: node linkType: hard +"@wdio/config@npm:8.10.1": + version: 8.10.1 + resolution: "@wdio/config@npm:8.10.1" + dependencies: + "@wdio/logger": 8.6.6 + "@wdio/types": 8.10.1 + "@wdio/utils": 8.10.1 + decamelize: ^6.0.0 + deepmerge-ts: ^5.0.0 + glob: ^10.2.2 + import-meta-resolve: ^3.0.0 + read-pkg-up: ^9.1.0 + checksum: 67ca55d65bae501878de2f25f65c687ecac3397f1951fb02c0b69a29901418b7c109debbb2501428b19ea729109404202d1ea4a7e15c8703de3d015e15d826dc + languageName: node + linkType: hard + "@wdio/globals@npm:8.10.0, @wdio/globals@npm:^8.1.3, @wdio/globals@npm:^8.8.8": version: 8.10.0 resolution: "@wdio/globals@npm:8.10.0" @@ -6229,6 +6278,15 @@ __metadata: languageName: node linkType: hard +"@wdio/repl@npm:8.10.1": + version: 8.10.1 + resolution: "@wdio/repl@npm:8.10.1" + dependencies: + "@types/node": ^20.1.0 + checksum: 7c770769e3db82f743f2dc9f604da8200f6eb7dfe4a708ed0b30e9c9b5c9c627342455991917c884d76448e4cc31054b85f9f843ba09c166faa32de9934571b3 + languageName: node + linkType: hard + "@wdio/repl@npm:8.6.6": version: 8.6.6 resolution: "@wdio/repl@npm:8.6.6" @@ -6252,6 +6310,20 @@ __metadata: languageName: node linkType: hard +"@wdio/reporter@npm:8.10.1": + version: 8.10.1 + resolution: "@wdio/reporter@npm:8.10.1" + dependencies: + "@types/node": ^20.1.0 + "@wdio/logger": 8.6.6 + "@wdio/types": 8.10.1 + diff: ^5.0.0 + object-inspect: ^1.12.0 + supports-color: 9.3.1 + checksum: 9d0016efa3a0701725ad1dc004955ca9fec588413dd6d4699c402e7337b439a48b37bd5f51a3e4b86bcaaa0e990e1c4334e17f7022c3a841659b47e6b714efed + languageName: node + linkType: hard + "@wdio/runner@npm:8.10.0": version: 8.10.0 resolution: "@wdio/runner@npm:8.10.0" @@ -6293,6 +6365,15 @@ __metadata: languageName: node linkType: hard +"@wdio/types@npm:8.10.1": + version: 8.10.1 + resolution: "@wdio/types@npm:8.10.1" + dependencies: + "@types/node": ^20.1.0 + checksum: c324aea065816c3246dfefa7f825ece25449c420bfa508f1776148aed93ebfc693649c6892e7b76c567982e37c6a133cfd9b61a0bb377d829aee952bf9e6d479 + languageName: node + linkType: hard + "@wdio/utils@npm:8.10.0": version: 8.10.0 resolution: "@wdio/utils@npm:8.10.0" @@ -6305,6 +6386,18 @@ __metadata: languageName: node linkType: hard +"@wdio/utils@npm:8.10.1": + version: 8.10.1 + resolution: "@wdio/utils@npm:8.10.1" + dependencies: + "@wdio/logger": 8.6.6 + "@wdio/types": 8.10.1 + import-meta-resolve: ^3.0.0 + p-iteration: ^1.1.8 + checksum: c8c3d41bd3fb125f5b2336c983f11f3a0af80f66740f43943810e24f2ca6f8c80a0186bc6a8ed3cc925759cd5a2bba61eda34cae6a04bb430928f281bd6827ac + languageName: node + linkType: hard + "@web3-storage/multipart-parser@npm:^1.0.0": version: 1.0.0 resolution: "@web3-storage/multipart-parser@npm:1.0.0" @@ -9145,6 +9238,13 @@ __metadata: languageName: node linkType: hard +"devtools-protocol@npm:0.0.1120988": + version: 0.0.1120988 + resolution: "devtools-protocol@npm:0.0.1120988" + checksum: 68eb7aa6a2fe20f8321168f9381849296b203355a5c052461b7ed95e8787b34458029dd64c8d4a8640d9fd329138a6d82f41237f5331ea4267c090dcbf6581f7 + languageName: node + linkType: hard + "devtools-protocol@npm:^0.0.1138159": version: 0.0.1138159 resolution: "devtools-protocol@npm:0.0.1138159" @@ -9174,6 +9274,28 @@ __metadata: languageName: node linkType: hard +"devtools@npm:8.10.1": + version: 8.10.1 + resolution: "devtools@npm:8.10.1" + dependencies: + "@types/node": ^20.1.0 + "@wdio/config": 8.10.1 + "@wdio/logger": 8.6.6 + "@wdio/protocols": 8.8.1 + "@wdio/types": 8.10.1 + "@wdio/utils": 8.10.1 + chrome-launcher: ^0.15.0 + edge-paths: ^3.0.5 + import-meta-resolve: ^3.0.0 + puppeteer-core: 20.1.1 + query-selector-shadow-dom: ^1.0.0 + ua-parser-js: ^1.0.1 + uuid: ^9.0.0 + which: ^3.0.0 + checksum: 2b8d91df314d7991d3189924ed5fcf1dbc8dec3b072980d612e0066296e256f7f38fc0cb35f799dd615b3b87503b207188b92c81be35b3cf519735a98068a1ae + languageName: node + linkType: hard + "diff-sequences@npm:^28.1.1": version: 28.1.1 resolution: "diff-sequences@npm:28.1.1" @@ -12678,6 +12800,17 @@ __metadata: languageName: node linkType: hard +"http-proxy-agent@npm:5.0.0, http-proxy-agent@npm:^5.0.0": + version: 5.0.0 + resolution: "http-proxy-agent@npm:5.0.0" + dependencies: + "@tootallnate/once": 2 + agent-base: 6 + debug: 4 + checksum: e2ee1ff1656a131953839b2a19cd1f3a52d97c25ba87bd2559af6ae87114abf60971e498021f9b73f9fd78aea8876d1fb0d4656aac8a03c6caa9fc175f22b786 + languageName: node + linkType: hard + "http-proxy-agent@npm:^4.0.0, http-proxy-agent@npm:^4.0.1": version: 4.0.1 resolution: "http-proxy-agent@npm:4.0.1" @@ -12689,17 +12822,6 @@ __metadata: languageName: node linkType: hard -"http-proxy-agent@npm:^5.0.0": - version: 5.0.0 - resolution: "http-proxy-agent@npm:5.0.0" - dependencies: - "@tootallnate/once": 2 - agent-base: 6 - debug: 4 - checksum: e2ee1ff1656a131953839b2a19cd1f3a52d97c25ba87bd2559af6ae87114abf60971e498021f9b73f9fd78aea8876d1fb0d4656aac8a03c6caa9fc175f22b786 - languageName: node - linkType: hard - "http-signature@npm:~1.2.0": version: 1.2.0 resolution: "http-signature@npm:1.2.0" @@ -18416,6 +18538,30 @@ __metadata: languageName: node linkType: hard +"puppeteer-core@npm:20.1.1": + version: 20.1.1 + resolution: "puppeteer-core@npm:20.1.1" + dependencies: + "@puppeteer/browsers": 1.0.1 + chromium-bidi: 0.4.7 + cross-fetch: 3.1.5 + debug: 4.3.4 + devtools-protocol: 0.0.1120988 + extract-zip: 2.0.1 + https-proxy-agent: 5.0.1 + proxy-from-env: 1.1.0 + tar-fs: 2.1.1 + unbzip2-stream: 1.4.3 + ws: 8.13.0 + peerDependencies: + typescript: ">= 4.7.4" + peerDependenciesMeta: + typescript: + optional: true + checksum: 60d1932c84b017694285568102360338c529e0cc7afe8a09e4ede3969db415ea6f3c115df009017876144a3a005a3e777a8d1b9e97baba25936f8fa44db55ebb + languageName: node + linkType: hard + "pvtsutils@npm:^1.3.2": version: 1.3.2 resolution: "pvtsutils@npm:1.3.2" @@ -22519,6 +22665,25 @@ __metadata: languageName: node linkType: hard +"webdriver@npm:8.10.1": + version: 8.10.1 + resolution: "webdriver@npm:8.10.1" + dependencies: + "@types/node": ^20.1.0 + "@types/ws": ^8.5.3 + "@wdio/config": 8.10.1 + "@wdio/logger": 8.6.6 + "@wdio/protocols": 8.8.1 + "@wdio/types": 8.10.1 + "@wdio/utils": 8.10.1 + deepmerge-ts: ^5.0.0 + got: ^12.1.0 + ky: ^0.33.0 + ws: ^8.8.0 + checksum: b30d161cd9698d0c91cf061e9a1f1b7cb88ef76019ade20d45caa069833e3c9d4e4df95e8f611f83f600f8b5af900130ef6c72b5d3424bb4331d9a6030abadec + languageName: node + linkType: hard + "webdriver@workspace:apps/webdriver": version: 0.0.0-use.local resolution: "webdriver@workspace:apps/webdriver" @@ -22575,6 +22740,39 @@ __metadata: languageName: node linkType: hard +"webdriverio@npm:8.10.1": + version: 8.10.1 + resolution: "webdriverio@npm:8.10.1" + dependencies: + "@types/node": ^20.1.0 + "@wdio/config": 8.10.1 + "@wdio/logger": 8.6.6 + "@wdio/protocols": 8.8.1 + "@wdio/repl": 8.10.1 + "@wdio/types": 8.10.1 + "@wdio/utils": 8.10.1 + archiver: ^5.0.0 + aria-query: ^5.0.0 + css-shorthand-properties: ^1.1.1 + css-value: ^0.0.1 + devtools: 8.10.1 + devtools-protocol: ^0.0.1138159 + grapheme-splitter: ^1.0.2 + import-meta-resolve: ^3.0.0 + is-plain-obj: ^4.1.0 + lodash.clonedeep: ^4.5.0 + lodash.zip: ^4.2.0 + minimatch: ^9.0.0 + puppeteer-core: 20.1.1 + query-selector-shadow-dom: ^1.0.0 + resq: ^1.9.1 + rgb2hex: 0.2.5 + serialize-error: ^8.0.0 + webdriver: 8.10.1 + checksum: a96e9e8395ebb20b64994595f3f1df2d9805432753d8d9725938cf46aa53869f1327d220bf9977286bd381f5798f3d1a370a87ca9dbcd877a7af6b265a78e093 + languageName: node + linkType: hard + "webidl-conversions@npm:^3.0.0": version: 3.0.1 resolution: "webidl-conversions@npm:3.0.1" @@ -23088,7 +23286,7 @@ __metadata: languageName: node linkType: hard -"yargs@npm:^17.5.1": +"yargs@npm:^17.5.1, yargs@npm:^17.7.2": version: 17.7.2 resolution: "yargs@npm:17.7.2" dependencies: diff --git a/scripts/e2e-run-ci b/scripts/e2e-run-ci deleted file mode 100755 index e8fa80283..000000000 --- a/scripts/e2e-run-ci +++ /dev/null @@ -1,14 +0,0 @@ -set -eux -export ENABLE_SSL=1 -export ENABLE_NETWORK_CACHING=1 -yarn workspace @tldraw/tldraw prebuild -SHELL=/bin/bash nohup yarn dev-webdriver & -PROCCESS_PID="$!" - -mode="${1:-local}" -sleep 5 -yarn workspace @tldraw/e2e "test:${mode}" -exit_code=$? - -kill $PROCCESS_PID -exit $exit_code \ No newline at end of file diff --git a/scripts/e2e-run-tests b/scripts/e2e-run-tests deleted file mode 100755 index d94285d00..000000000 --- a/scripts/e2e-run-tests +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash -set -eux - -mode="${1:-local}" -export BROWSERS="${2:-chrome}" -# if [[ "$mode" == "remote" ]]; then -# export ENABLE_SSL=1 -# fi -yarn workspace @tldraw/e2e "test:${mode}" diff --git a/scripts/e2e-start-server b/scripts/e2e-start-server deleted file mode 100755 index 618cadcd7..000000000 --- a/scripts/e2e-start-server +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash -set -eux -yarn dev-webdriver diff --git a/scripts/e2e/commands/index.ts b/scripts/e2e/commands/index.ts new file mode 100644 index 000000000..c894ffc30 --- /dev/null +++ b/scripts/e2e/commands/index.ts @@ -0,0 +1,7 @@ +import seleniumGrid from './selenium-grid' +import serve from './serve' +import testBrowserstack from './test-browserstack' +import testCi from './test-ci' +import testLocal from './test-local' + +export { testCi, testLocal, testBrowserstack, seleniumGrid, serve } diff --git a/scripts/e2e/commands/selenium-grid.ts b/scripts/e2e/commands/selenium-grid.ts new file mode 100644 index 000000000..90c50ae3c --- /dev/null +++ b/scripts/e2e/commands/selenium-grid.ts @@ -0,0 +1,36 @@ +import { promiseSpawn } from './util' + +export default async function seleniumGrid() { + // NOTE: This should work on non-macos, but it's only be tested on macos with M1 chipset + const command = 'docker' + let args: string[] = [] + if (process.arch === 'arm64') { + args = [ + `run`, + `-t`, + `--platform`, + `linux/amd64`, + `-p`, + `4444:4444`, + `-p`, + `7900:7900`, + `--shm-size=2g`, + `seleniarm/standalone-firefox:latest`, + ] + } else { + args = [ + 'run', + '-t', + '-p', + '4444:4444', + '-p', + '7900:7900', + `--shm-size=2g`, + `selenium/standalone-firefox:latest`, + ] + } + + return promiseSpawn(command, args, { + stdio: [0, 0, 0], // Use parent's [stdin, stdout, stderr] + }) +} diff --git a/scripts/e2e/commands/serve.ts b/scripts/e2e/commands/serve.ts new file mode 100644 index 000000000..23dff7be8 --- /dev/null +++ b/scripts/e2e/commands/serve.ts @@ -0,0 +1,8 @@ +import { promiseSpawn } from './util' + +export default async function serve() { + return promiseSpawn('yarn', ['dev-webdriver'], { + stdio: [0, 0, 0], + env: { ...process.env }, + }) +} diff --git a/scripts/e2e/commands/test-browserstack.ts b/scripts/e2e/commands/test-browserstack.ts new file mode 100644 index 000000000..506fb1ad7 --- /dev/null +++ b/scripts/e2e/commands/test-browserstack.ts @@ -0,0 +1,20 @@ +import { promiseSpawn } from './util' + +export default async function testBrowserstack({ + os, + browser, +}: { + os: string[] + browser: string[] +}) { + const command = `yarn` + const args = [`workspace`, `@tldraw/e2e`, `test:remote`] + return promiseSpawn(command, args, { + env: { + ...process.env, + BROWSERS: browser.join(','), + OS: os.join(','), + }, + stdio: [0, 0, 0], // Use parent's [stdin, stdout, stderr] + }) +} diff --git a/scripts/e2e/commands/test-ci.ts b/scripts/e2e/commands/test-ci.ts new file mode 100644 index 000000000..0f50c5d40 --- /dev/null +++ b/scripts/e2e/commands/test-ci.ts @@ -0,0 +1,60 @@ +import { ChildProcess, spawn } from 'node:child_process' +import kill from 'tree-kill' +import { exec } from '../../lib/exec' +import { promiseSpawn } from './util' + +export default async function testCi({ testEnv }: { testEnv: string }) { + await promiseSpawn('yarn', ['workspace', '@tldraw/tldraw', 'prebuild'], { + env: { + ...process.env, + }, + stdio: [0, 0, 0], // Use parent's [stdin, stdout, stderr] + }) + + const { success: foundStartMessage, commandProcess } = await new Promise<{ + success: boolean + commandProcess: ChildProcess + }>((resolve, reject) => { + const p = spawn('yarn', ['dev-webdriver'], { + env: { + ...process.env, + ENABLE_SSL: '1', + ENABLE_NETWORK_CACHING: '1', + }, + }) + + const endHandler = () => { + p.stdout.off('end', endHandler) + reject({ success: false, commandProcess: p }) + } + + const dataHandler = (data: any) => { + if (data.toString().match(/\[tldraw:process_ready\]/gm)) { + // p.stdout.off('data', dataHandler) + resolve({ success: true, commandProcess: p }) + } + console.log(`stdout: ${data}`) + } + p.stdout.on('data', dataHandler) + p.stdout.on('close', endHandler) + }) + + if (!foundStartMessage) { + console.error('Failed to start server') + process.exit(1) + } + + const exitCode = await exec('yarn', ['workspace', '@tldraw/e2e', `test:${testEnv}`], { + env: { + ...process.env, + BROWSERS: ['chrome'].join(','), + // OS: [process.platform].join(','), + }, + }) + + if (commandProcess.pid) { + kill(commandProcess.pid) + } + + return exitCode +} diff --git a/scripts/e2e/commands/test-local.ts b/scripts/e2e/commands/test-local.ts new file mode 100644 index 000000000..9b44274ae --- /dev/null +++ b/scripts/e2e/commands/test-local.ts @@ -0,0 +1,14 @@ +import { promiseSpawn } from './util' + +export default async function testLocal({ os, browser }: { os: string; browser: string[] }) { + const command = `yarn` + const args = [`workspace`, `@tldraw/e2e`, `test:local`] + return promiseSpawn(command, args, { + env: { + ...process.env, + BROWSERS: browser.join(','), + OS: os, + }, + stdio: [0, 0, 0], // Use parent's [stdin, stdout, stderr] + }) +} diff --git a/scripts/e2e/commands/util.ts b/scripts/e2e/commands/util.ts new file mode 100644 index 000000000..e34849767 --- /dev/null +++ b/scripts/e2e/commands/util.ts @@ -0,0 +1,16 @@ +import { SpawnOptions, spawn } from 'child_process' +import kill from 'tree-kill' + +export function promiseSpawn(command: string, args: string[], opts: SpawnOptions) { + return new Promise((resolve) => { + const p = spawn(command, args, opts) + p.on('close', (exitCode) => { + resolve(exitCode ?? 0) + }) + process.on('SIGINT', () => { + if (p.pid) { + kill(p.pid) + } + }) + }) +} diff --git a/scripts/e2e/index.ts b/scripts/e2e/index.ts new file mode 100644 index 000000000..56d138cd3 --- /dev/null +++ b/scripts/e2e/index.ts @@ -0,0 +1,94 @@ +import { hideBin } from 'yargs/helpers' +import yargs from 'yargs/yargs' +import * as commands from './commands' + +yargs(hideBin(process.argv)) + .usage('Usage: $0 [options]') + .scriptName('yarn e2e') + .command( + 'serve', + 'start test server', + (yargs) => { + return yargs + }, + async () => { + const exitCode = await commands.serve() + process.exit(exitCode) + } + ) + .command( + 'test:ci [env]', + 'runner for CI (github-actions)', + (yargs) => { + return yargs.positional('env', { + type: 'string', + default: 'local', + choices: ['local', 'nightly'], + }) + }, + async (argv) => { + await commands.testCi({ testEnv: argv.env }) + // process.exit(exitCode) + } + ) + .command( + 'test:local', + 'run webdriver tests locally', + (yargs) => { + return yargs + .option('browser', { + alias: 'b', + type: 'array', + description: 'run with browsers', + choices: ['chrome', 'firefox', 'safari', 'edge', 'vscode'], + default: ['chrome'], + }) + .option('os', { + type: 'string', + description: 'OS to run on (experimental)', + choices: [process.platform, 'linux'], + default: process.platform, + }) + }, + async (argv) => { + const exitCode = await commands.testLocal(argv) + process.exit(exitCode) + } + ) + .command( + 'test:browserstack', + 'run webdriver tests on browserstack', + (yargs) => { + return yargs + .option('browser', { + alias: 'b', + type: 'array', + description: 'run with browsers', + choices: ['chrome', 'firefox', 'safari', 'edge'], + default: ['chrome'], + }) + .option('os', { + type: 'array', + description: 'OS to run on (experimental)', + choices: [process.platform, 'linux'], + default: [process.platform], + }) + }, + async (argv) => { + const exitCode = await commands.testBrowserstack(argv) + process.exit(exitCode) + } + ) + .command( + 'selenium:grid', + 'start selenium grid (test linux)', + (yargs) => { + return yargs + }, + async () => { + const exitCode = await commands.seleniumGrid() + process.exit(exitCode) + } + ) + .strict() + .parse()