chore: bring in folio source (#6923)

- Source now lives at `src/test`.
- Former folio tests live at `tests/playwright-test`.
- We use `src/test/internal.ts` that exposes base test without
  Playwright fixtures for most tests (to avoid modifications for now).
- Test types live in `types/testFoo.d.ts`.
- Stable test runner is installed to `tests/config/test-runner` during `npm install`.
- All deps including test-only are now listed in `package.json`.
  Non-test deps must also be listed in `build_package.js` to get included.

Author: dgozman

.github/workflows/tests_primary.yml

@@ -47,3 +47,20 @@ jobs:
         name: ${{ matrix.browser }}-${{ matrix.os }}-test-results
         path: test-results
 
+  test_test_runner:
+    name: Test Runner
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-node@v2
+      with:
+        node-version: 12
+    - run: npm ci
+      env:
+        DEBUG: pw:install
+    - run: npm run build
+    - run: npm run ttest

install-from-github.js

@@ -18,6 +18,17 @@
 // This file is only run when someone installs via the github repo
 
 const {execSync} = require('child_process');
+const path = require('path');
+
+console.log(`Updating test runner...`);
+try {
+  execSync('npm ci --save=false --fund=false --audit=false', {
+    stdio: ['inherit', 'inherit', 'inherit'],
+    cwd: path.join(__dirname, 'tests', 'config', 'test-runner'),
+  });
+} catch (e) {
+  process.exit(1);
+}
 
 console.log(`Rebuilding installer...`);
 try {

package-lock.json

@@ -9,12 +9,14 @@
     "node": ">=12"
   },
   "scripts": {
-    "ctest": "folio --config=tests/config/default.config.ts --project=chromium",
-    "ftest": "folio --config=tests/config/default.config.ts --project=firefox",
-    "wtest": "folio --config=tests/config/default.config.ts --project=webkit",
-    "atest": "folio --config=tests/config/android.config.ts",
-    "etest": "folio --config=tests/config/electron.config.ts",
-    "test": "folio --config=tests/config/default.config.ts",
+    "basetest": "node ./tests/config/test-runner/node_modules/@playwright/test/lib/cli/cli.js test",
+    "ctest": "npm run basetest -- --config=tests/config/default.config.ts --project=chromium",
+    "ftest": "npm run basetest -- --config=tests/config/default.config.ts --project=firefox",
+    "wtest": "npm run basetest -- --config=tests/config/default.config.ts --project=webkit",
+    "atest": "npm run basetest -- --config=tests/config/android.config.ts",
+    "etest": "npm run basetest -- --config=tests/config/electron.config.ts",
+    "ttest": "npm run basetest -- --config=tests/playwright-test/playwright-test.config.ts",
+    "test": "npm run basetest -- --config=tests/config/default.config.ts",
     "eslint": "[ \"$CI\" = true ] && eslint --quiet -f codeframe --ext ts . || eslint --ext ts .",
     "tsc": "tsc -p .",
     "tsc-installer": "tsc -p ./src/install/tsconfig.json",
@@ -36,7 +38,32 @@
   "bin": {
     "playwright": "./lib/cli/cli.js"
   },
+  "DEPS-NOTE": "Any non-test dependency must be added to the build_package.js script as well",
   "dependencies": {
+    "@babel/code-frame": "^7.12.13",
+    "@babel/core": "^7.14.0",
+    "@babel/plugin-proposal-class-properties": "^7.13.0",
+    "@babel/plugin-proposal-dynamic-import": "^7.13.8",
+    "@babel/plugin-proposal-export-namespace-from": "^7.12.13",
+    "@babel/plugin-proposal-logical-assignment-operators": "^7.13.8",
+    "@babel/plugin-proposal-nullish-coalescing-operator": "^7.13.8",
+    "@babel/plugin-proposal-numeric-separator": "^7.12.13",
+    "@babel/plugin-proposal-optional-chaining": "^7.13.12",
+    "@babel/plugin-proposal-private-methods": "^7.13.0",
+    "@babel/plugin-proposal-private-property-in-object": "^7.14.0",
+    "@babel/plugin-syntax-async-generators": "^7.8.4",
+    "@babel/plugin-syntax-json-strings": "^7.8.3",
+    "@babel/plugin-syntax-object-rest-spread": "^7.8.3",
+    "@babel/plugin-syntax-optional-catch-binding": "^7.8.3",
+    "@babel/plugin-transform-modules-commonjs": "^7.14.0",
+    "@babel/preset-typescript": "^7.13.0",
+    "colors": "^1.4.0",
+    "expect": "^26.4.2",
+    "minimatch": "^3.0.3",
+    "ms": "^2.1.2",
+    "pirates": "^4.0.1",
+    "pixelmatch": "^5.2.1",
+    "source-map-support": "^0.4.18",
     "commander": "^6.1.0",
     "debug": "^4.1.1",
     "extract-zip": "^2.0.1",
@@ -53,10 +80,14 @@
     "yazl": "^2.5.1"
   },
   "devDependencies": {
+    "@types/babel__code-frame": "^7.0.2",
+    "@types/babel__core": "^7.1.14",
     "@types/debug": "^4.1.5",
     "@types/extract-zip": "^1.6.2",
     "@types/mime": "^2.0.3",
+    "@types/minimatch": "^3.0.3",
     "@types/node": "^10.17.28",
+    "@types/pixelmatch": "^5.2.1",
     "@types/pngjs": "^3.4.2",
     "@types/progress": "^2.0.3",
     "@types/proper-lockfile": "^4.1.1",
@@ -65,8 +96,10 @@
     "@types/react-dom": "^17.0.0",
     "@types/resize-observer-browser": "^0.1.4",
     "@types/rimraf": "^3.0.0",
+    "@types/source-map-support": "^0.4.2",
     "@types/webpack": "^4.41.25",
     "@types/ws": "7.2.6",
+    "@types/xml2js": "^0.4.5",
     "@types/yazl": "^2.4.2",
     "@typescript-eslint/eslint-plugin": "^4.25.0",
     "@typescript-eslint/parser": "^4.25.0",
@@ -80,7 +113,6 @@
     "eslint-plugin-notice": "^0.9.10",
     "eslint-plugin-react-hooks": "^4.2.0",
     "file-loader": "^6.1.0",
-    "folio": "=0.4.0-alpha28",
     "formidable": "^1.2.2",
     "html-webpack-plugin": "^4.4.1",
     "ncp": "^2.0.0",
@@ -90,9 +122,10 @@
     "socksv5": "0.0.6",
     "style-loader": "^1.2.1",
     "ts-loader": "^8.0.3",
-    "typescript": "^4.0.2",
+    "typescript": "=4.2.4",
     "webpack": "^4.44.2",
     "webpack-cli": "^3.3.12",
+    "xml2js": "^0.4.23",
     "yaml": "^1.10.0"
   }
 }

package.json

@@ -61,10 +61,27 @@ const PACKAGES = {
   'playwright-chromium': {
     description: 'A high-level API to automate Chromium',
     browsers: ['chromium', 'ffmpeg'],
-    files: [...PLAYWRIGHT_CORE_FILES],
+    files: PLAYWRIGHT_CORE_FILES,
   },
 };
 
+const DEPENDENCIES = [
+  'commander',
+  'debug',
+  'extract-zip',
+  'https-proxy-agent',
+  'jpeg-js',
+  'mime',
+  'pngjs',
+  'progress',
+  'proper-lockfile',
+  'proxy-from-env',
+  'rimraf',
+  'stack-utils',
+  'ws',
+  'yazl',
+];
+
 // 1. Parse CLI arguments
 const args = process.argv.slice(2);
 if (args.some(arg => arg === '--help')) {
@@ -121,9 +138,10 @@ if (!args.some(arg => arg === '--no-cleanup')) {
 
   // 4. Generate package.json
   const pwInternalJSON = require(path.join(ROOT_PATH, 'package.json'));
-  const dependencies = { ...pwInternalJSON.dependencies };
-  if (packageName === 'playwright-test')
-    dependencies.folio = pwInternalJSON.devDependencies.folio;
+  const depNames = packageName === 'playwright-test' ? Object.keys(pwInternalJSON.dependencies) : DEPENDENCIES;
+  const dependencies = {};
+  for (const dep of depNames)
+    dependencies[dep] =  pwInternalJSON.dependencies[dep];
   await writeToPackage('package.json', JSON.stringify({
     name: package.name || packageName,
     version: pwInternalJSON.version,

packages/build_package.js

@@ -16,5 +16,5 @@
 
 module.exports = {
   ...require('./lib/inprocess'),
-  ...require('./lib/cli/fixtures')
+  ...require('./lib/test/index')
 };

packages/playwright-test/index.js

@@ -35,16 +35,16 @@ import { BrowserContextOptions, LaunchOptions } from '../client/types';
 import { spawn } from 'child_process';
 import { installDeps } from '../install/installDeps';
 import { allBrowserNames, BrowserName } from '../utils/registry';
-import { addTestCommand } from './testRunner';
 import * as utils from '../utils/utils';
 
 const SCRIPTS_DIRECTORY = path.join(__dirname, '..', '..', 'bin');
 
 type BrowserChannel = 'chrome-beta'|'chrome';
 const allBrowserChannels: Set<BrowserChannel> = new Set(['chrome-beta', 'chrome']);
+const packageJSON = require('../../package.json');
 
 program
-    .version('Version ' + require('../../package.json').version)
+    .version('Version ' + packageJSON.version)
     .name(process.env.PW_CLI_NAME || 'npx playwright');
 
 commandWithOpenOptions('open [url]', 'open page in browser specified via -b, --browser', [])
@@ -226,8 +226,19 @@ program
       console.log('  $ show-trace trace/directory');
     });
 
-if (!process.env.PW_CLI_TARGET_LANG)
-  addTestCommand(program);
+if (!process.env.PW_CLI_TARGET_LANG) {
+  if (packageJSON.name === '@playwright/test' || process.env.PWTEST_CLI_ALLOW_TEST_COMMAND) {
+    require('../test/cli').addTestCommand(program);
+  } else {
+    const command = program.command('test');
+    command.description('Run tests with Playwright Test. Available in @playwright/test package.');
+    command.action(async (args, opts) => {
+      console.error('Please install @playwright/test package to use Playwright Test.');
+      console.error('  npm install -D @playwright/test');
+      process.exit(1);
+    });
+  }
+}
 
 if (process.argv[2] === 'run-driver')
   runDriver();

src/cli/cli.ts

@@ -19,9 +19,8 @@
 import * as commander from 'commander';
 import * as fs from 'fs';
 import * as path from 'path';
-import type { Config } from 'folio';
-
-type RunnerType = typeof import('folio/out/runner').Runner;
+import type { Config } from './types';
+import { Runner } from './runner';
 
 const defaultTimeout = 30000;
 const defaultReporter = process.env.CI ? 'dot' : 'list';
@@ -37,14 +36,6 @@ const defaultConfig: Config = {
 };
 
 export function addTestCommand(program: commander.CommanderStatic) {
-  let Runner: RunnerType;
-  try {
-    Runner = require('folio/out/runner').Runner as RunnerType;
-  } catch (e) {
-    addStubTestCommand(program);
-    return;
-  }
-
   const command = program.command('test [test-filter...]');
   command.description('Run tests with Playwright Test');
   command.option('--browser <browser>', `Browser to use for tests, one of "all", "chromium", "firefox" or "webkit" (default: "chromium")`);
@@ -68,7 +59,7 @@ export function addTestCommand(program: commander.CommanderStatic) {
   command.option('-x', `Stop after the first failure`);
   command.action(async (args, opts) => {
     try {
-      await runTests(Runner, args, opts);
+      await runTests(args, opts);
     } catch (e) {
       console.error(e.toString());
       process.exit(1);
@@ -86,7 +77,7 @@ export function addTestCommand(program: commander.CommanderStatic) {
   });
 }
 
-async function runTests(Runner: RunnerType, args: string[], opts: { [key: string]: any }) {
+async function runTests(args: string[], opts: { [key: string]: any }) {
   const browserOpt = opts.browser ? opts.browser.toLowerCase() : 'chromium';
   if (!['all', 'chromium', 'firefox', 'webkit'].includes(browserOpt))
     throw new Error(`Unsupported browser "${opts.browser}", must be one of "all", "chromium", "firefox" or "webkit"`);
@@ -135,11 +126,6 @@ async function runTests(Runner: RunnerType, args: string[], opts: { [key: string
     throw new Error(`Configuration file not found. Run "npx playwright test --help" for more information.`);
   }
 
-  process.env.FOLIO_JUNIT_OUTPUT_NAME = process.env.PLAYWRIGHT_JUNIT_OUTPUT_NAME;
-  process.env.FOLIO_JUNIT_SUITE_ID = process.env.PLAYWRIGHT_JUNIT_SUITE_ID;
-  process.env.FOLIO_JUNIT_SUITE_NAME = process.env.PLAYWRIGHT_JUNIT_SUITE_NAME;
-  process.env.FOLIO_JSON_OUTPUT_NAME = process.env.PLAYWRIGHT_JSON_OUTPUT_NAME;
-
   const result = await runner.run(!!opts.list, args.map(forceRegExp), opts.project || undefined);
   if (result === 'sigint')
     process.exit(130);
@@ -172,13 +158,3 @@ function overridesFromOptions(options: { [key: string]: any }): Config {
     workers: options.workers ? parseInt(options.workers, 10) : undefined,
   };
 }
-
-function addStubTestCommand(program: commander.CommanderStatic) {
-  const command = program.command('test');
-  command.description('Run tests with Playwright Test. Available in @playwright/test package.');
-  command.action(async (args, opts) => {
-    console.error('Please install @playwright/test package to use Playwright Test.');
-    console.error('  npm install -D @playwright/test');
-    process.exit(1);
-  });
-}