test: roll to [email protected] (#6789)

Author: dgozman

.github/workflows/tests_secondary.yml

@@ -148,8 +148,10 @@ jobs:
     - run: node lib/cli/cli install-deps ${{ matrix.browser }} chromium
     # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
     # Wrap `npm run` in a subshell to redirect STDERR to file.
-    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "npm run test -- --project=${{ matrix.browser }} --headed"
+    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "npm run test -- --project=${{ matrix.browser }}"
       if: ${{ always() }}
+      env:
+        HEADFUL: 1
     - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
       if: always()
     - uses: actions/upload-artifact@v1
@@ -177,7 +179,9 @@ jobs:
     - run: node lib/cli/cli install-deps chromium
     # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
     # Wrap `npm run` in a subshell to redirect STDERR to file.
-    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "npm run ctest -- --mode=${{ matrix.mode }}"
+    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "npm run ctest"
+      env:
+        PWTEST_MODE: ${{ matrix.mode }}
     - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
       if: always()
     - uses: actions/upload-artifact@v1
@@ -205,7 +209,9 @@ jobs:
     - run: node lib/cli/cli install-deps ${{ matrix.browser }} chromium
     # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
     # Wrap `npm run` in a subshell to redirect STDERR to file.
-    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "npm run test -- --project=${{ matrix.browser }} --video"
+    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "npm run test -- --project=${{ matrix.browser }}"
+      env:
+        PWTEST_VIDEO: 1
     - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
       if: always()
     - uses: actions/upload-artifact@v1
@@ -264,7 +270,9 @@ jobs:
     - run: node lib/cli/cli install ffmpeg
     # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
     # Wrap `npm run` in a subshell to redirect STDERR to file.
-    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "npm run ctest -- --channel=chrome"
+    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "npm run ctest"
+      env:
+        PWTEST_CHANNEL: chrome
     - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
       if: always()
     - uses: actions/upload-artifact@v1
@@ -290,8 +298,10 @@ jobs:
     - run: npm run build
     # This only created problems, should we move ffmpeg back into npm?
     - run: node lib/cli/cli install ffmpeg
-    - run: npm run ctest -- --channel=chrome
+    - run: npm run ctest
       shell: bash
+      env:
+        PWTEST_CHANNEL: chrome
     - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
       if: always()
       shell: bash
@@ -315,7 +325,9 @@ jobs:
     - run: npm run build
     # This only created problems, should we move ffmpeg back into npm?
     - run: node lib/cli/cli install ffmpeg
-    - run: npm run ctest -- --channel=chrome
+    - run: npm run ctest
+      env:
+        PWTEST_CHANNEL: chrome
     - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
       if: always()
     - uses: actions/upload-artifact@v1
@@ -340,7 +352,9 @@ jobs:
     - run: node lib/cli/cli install ffmpeg firefox-stable chromium
     # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
     # Wrap `npm run` in a subshell to redirect STDERR to file.
-    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "npm run ftest -- --channel=firefox-stable"
+    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "npm run ftest"
+      env:
+        PWTEST_CHANNEL: firefox-stable
     - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
       if: always()
     - uses: actions/upload-artifact@v1
@@ -365,8 +379,10 @@ jobs:
         PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
     - run: npm run build
     - run: node lib/cli/cli install ffmpeg firefox-stable chromium
-    - run: npm run ftest -- --channel=firefox-stable
+    - run: npm run ftest
       shell: bash
+      env:
+        PWTEST_CHANNEL: firefox-stable
     - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
       if: always()
       shell: bash
@@ -389,7 +405,9 @@ jobs:
         PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
     - run: npm run build
     - run: node lib/cli/cli install ffmpeg firefox-stable chromium
-    - run: npm run ftest -- --channel=firefox-stable
+    - run: npm run ftest
+      env:
+        PWTEST_CHANNEL: firefox-stable
     - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
       if: always()
     - uses: actions/upload-artifact@v1
@@ -415,8 +433,10 @@ jobs:
     - run: npm run build
     # This only created problems, should we move ffmpeg back into npm?
     - run: node lib/cli/cli install ffmpeg
-    - run: npm run ctest -- --channel=msedge
+    - run: npm run ctest
       shell: bash
+      env:
+        PWTEST_CHANNEL: msedge
     - uses: actions/upload-artifact@v1
       if: ${{ always() }}
       with:
@@ -463,7 +483,9 @@ jobs:
     - run: node lib/cli/cli install ffmpeg
     # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
     # Wrap `npm run` in a subshell to redirect STDERR to file.
-    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "npm run ctest -- --channel=chrome-beta"
+    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "npm run ctest"
+      env:
+        PWTEST_CHANNEL: chrome-beta
     - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
       if: always()
     - uses: actions/upload-artifact@v1
@@ -490,8 +512,10 @@ jobs:
         PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
     - run: npm run build
     - run: node lib/cli/cli install ffmpeg
-    - run: npm run ctest -- --channel=chrome-beta
+    - run: npm run ctest
       shell: bash
+      env:
+        PWTEST_CHANNEL: chrome-beta
     - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
       if: always()
       shell: bash
@@ -515,7 +539,9 @@ jobs:
         PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
     - run: npm run build
     - run: node lib/cli/cli install ffmpeg
-    - run: npm run ctest -- --channel=chrome-beta
+    - run: npm run ctest
+      env:
+        PWTEST_CHANNEL: chrome-beta
     - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
       if: always()
     - uses: actions/upload-artifact@v1

package-lock.json

@@ -80,7 +80,7 @@
     "eslint-plugin-notice": "^0.9.10",
     "eslint-plugin-react-hooks": "^4.2.0",
     "file-loader": "^6.1.0",
-    "folio": "=0.4.0-alpha19",
+    "folio": "=0.4.0-alpha21",
     "formidable": "^1.2.2",
     "html-webpack-plugin": "^4.4.1",
     "ncp": "^2.0.0",

package.json

@@ -25,7 +25,6 @@ const outputDir = path.join(__dirname, '..', '..', 'test-results');
 const testDir = path.join(__dirname, '..');
 const config: folio.Config<CommonOptions & PlaywrightOptions> = {
   testDir,
-  snapshotDir: '__snapshots__',
   outputDir,
   timeout: 120000,
   globalTimeout: 7200000,

tests/__snapshots__/chromium/oopif/should-take-screenshot/chromium/screenshot-oopif.png renamed to tests/chromium/oopif.spec.ts-snapshots/should-take-screenshot-chromium-screenshot-oopif.png

@@ -124,7 +124,7 @@ const baseFixtures: folio.Fixtures<{ __baseSetup: void }, BaseOptions & BaseFixt
   isMac: [ process.platform === 'darwin', { scope: 'worker' } ],
   isLinux: [ process.platform === 'linux', { scope: 'worker' } ],
   __baseSetup: [ async ({ browserName }, run, testInfo) => {
-    testInfo.snapshotPathSegment = browserName;
+    testInfo.snapshotSuffix = browserName;
     await run();
   }, { auto: true } ],
 };

tests/config/android.config.ts

@@ -38,16 +38,15 @@ const pageFixtures = {
   isElectron: false,
 };
 
-const mode = (folio.registerCLIOption('mode', 'Transport mode: default, driver or service').value || 'default') as ('default' | 'driver' | 'service');
-const headed = folio.registerCLIOption('headed', 'Run tests in headed mode (default: headless)', { type: 'boolean' }).value || !!process.env.HEADFUL;
-const channel = folio.registerCLIOption('channel', 'Browser channel (default: no channel)').value as any;
-const video = !!folio.registerCLIOption('video', 'Record videos for all tests', { type: 'boolean' }).value;
+const mode = (process.env.PWTEST_MODE || 'default') as ('default' | 'driver' | 'service');
+const headed = !!process.env.HEADFUL;
+const channel = process.env.PWTEST_CHANNEL as any;
+const video = !!process.env.PWTEST_VIDEO;
 
 const outputDir = path.join(__dirname, '..', '..', 'test-results');
 const testDir = path.join(__dirname, '..');
 const config: folio.Config<CommonOptions & PlaywrightOptions> = {
   testDir,
-  snapshotDir: '__snapshots__',
   outputDir,
   timeout: video || process.env.PWTRACE ? 60000 : 30000,
   globalTimeout: 5400000,

tests/config/baseTest.ts

@@ -25,7 +25,6 @@ const outputDir = path.join(__dirname, '..', '..', 'test-results');
 const testDir = path.join(__dirname, '..');
 const config: folio.Config<CommonOptions & PlaywrightOptions> = {
   testDir,
-  snapshotDir: '__snapshots__',
   outputDir,
   timeout: 30000,
   globalTimeout: 5400000,