test(tracing): start adding tracing tests (#6369)

Author: pavelfeldman

src/server/snapshot/snapshotter.ts

@@ -52,6 +52,10 @@ export class Snapshotter {
     this._snapshotStreamer = '__playwright_snapshot_streamer_' + guid;
   }
 
+  started(): boolean {
+    return this._started;
+  }
+
   async start() {
     this._started = true;
     if (!this._initialized) {

src/server/trace/recorder/traceSnapshotter.ts

@@ -42,6 +42,10 @@ export class TraceSnapshotter extends EventEmitter implements SnapshotterDelegat
     this._writeArtifactChain = Promise.resolve();
   }
 
+  started(): boolean {
+    return this._snapshotter.started();
+  }
+
   async start(): Promise<void> {
     await this._snapshotter.start();
   }

src/server/trace/recorder/tracing.ts

@@ -118,13 +118,13 @@ export class Tracing implements InstrumentationListener {
     const zipFile = new yazl.ZipFile();
     zipFile.addFile(this._traceFile, 'trace.trace');
     const zipFileName = this._traceFile + '.zip';
+    this._traceFile = undefined;
     for (const sha1 of this._sha1s)
       zipFile.addFile(path.join(this._resourcesDir!, sha1), path.join('resources', sha1));
-    const zipPromise = new Promise(f => {
+    zipFile.end();
+    await new Promise(f => {
       zipFile.outputStream.pipe(fs.createWriteStream(zipFileName)).on('close', f);
     });
-    zipFile.end();
-    await zipPromise;
     const artifact = new Artifact(this._context, zipFileName);
     artifact.reportFinished();
     return artifact;
@@ -133,7 +133,7 @@ export class Tracing implements InstrumentationListener {
   async _captureSnapshot(name: 'before' | 'after' | 'action' | 'event', sdkObject: SdkObject, metadata: CallMetadata, element?: ElementHandle) {
     if (!sdkObject.attribution.page)
       return;
-    if (!this._snapshotter)
+    if (!this._snapshotter.started())
       return;
     const snapshotName = `${name}@${metadata.id}`;
     metadata.snapshots.push({ title: name, snapshotName });

tests/config/browserTest.ts

@@ -209,5 +209,6 @@ class ContextEnv {
 }
 
 export const contextTest = browserTest.extend(new ContextEnv());
+export const tracingTest = baseTest.extend(new PlaywrightEnv()).extend(new BrowserEnv()).extend(new ContextEnv());
 
 export { expect } from 'folio';

tests/config/default.config.ts

@@ -16,7 +16,7 @@
 
 import * as folio from 'folio';
 import * as path from 'path';
-import { playwrightTest, slowPlaywrightTest, contextTest } from './browserTest';
+import { playwrightTest, slowPlaywrightTest, contextTest, tracingTest } from './browserTest';
 import { test as pageTest } from './pageTest';
 import { BrowserName, CommonTestArgs, CommonWorkerArgs } from './baseTest';
 import type { Browser, BrowserContext } from '../../index';
@@ -115,4 +115,5 @@ for (const browserName of browsers) {
   playwrightTest.runWith(envConfig);
   slowPlaywrightTest.runWith({ ...envConfig, timeout: config.timeout * 3 });
   pageTest.runWith(envConfig, new PageEnv());
+  tracingTest.runWith({ options: { ...envConfig.options, traceDir: path.join(config.outputDir, 'trace-' + process.env.FOLIO_WORKER_INDEX) }, tag: browserName });
 }

tests/tracing.spec.ts

@@ -0,0 +1,130 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import path from 'path';
+import { expect, tracingTest as test } from './config/browserTest';
+import yauzl from 'yauzl';
+import removeFolder from 'rimraf';
+
+test.beforeEach(async ({}, testInfo) => {
+  const folder = path.join(testInfo.config.outputDir, 'trace-' + process.env.FOLIO_WORKER_INDEX);
+  await new Promise(f => removeFolder(folder, f));
+});
+
+test('should collect trace', async ({ context, page, server, browserName }, testInfo) => {
+  await (context as any)._tracing.start({ name: 'test', screenshots: true, snapshots: true });
+  await page.goto(server.EMPTY_PAGE);
+  await page.setContent('<button>Click</button>');
+  await page.click('"Click"');
+  await page.close();
+  await (context as any)._tracing.stop();
+  await (context as any)._tracing.export(testInfo.outputPath('trace.zip'));
+
+  const { events } = await parseTrace(testInfo.outputPath('trace.zip'));
+  expect(events[0].type).toBe('context-metadata');
+  expect(events[1].type).toBe('page-created');
+  expect(events.find(e => e.metadata?.apiName === 'page.goto')).toBeTruthy();
+  expect(events.find(e => e.metadata?.apiName === 'page.setContent')).toBeTruthy();
+  expect(events.find(e => e.metadata?.apiName === 'page.click')).toBeTruthy();
+  expect(events.find(e => e.metadata?.apiName === 'page.close')).toBeTruthy();
+
+  expect(events.some(e => e.type === 'frame-snapshot')).toBeTruthy();
+  expect(events.some(e => e.type === 'resource-snapshot')).toBeTruthy();
+  if (browserName === 'chromium')
+    expect(events.some(e => e.type === 'screencast-frame')).toBeTruthy();
+});
+
+test('should collect trace', async ({ context, page, server }, testInfo) => {
+  await (context as any)._tracing.start({ name: 'test' });
+  await page.goto(server.EMPTY_PAGE);
+  await page.setContent('<button>Click</button>');
+  await page.click('"Click"');
+  await page.close();
+  await (context as any)._tracing.stop();
+  await (context as any)._tracing.export(testInfo.outputPath('trace.zip'));
+
+  const { events } = await parseTrace(testInfo.outputPath('trace.zip'));
+  expect(events.some(e => e.type === 'frame-snapshot')).toBeFalsy();
+  expect(events.some(e => e.type === 'resource-snapshot')).toBeFalsy();
+});
+
+test('should collect two traces', async ({ context, page, server }, testInfo) => {
+  await (context as any)._tracing.start({ name: 'test1', screenshots: true, snapshots: true });
+  await page.goto(server.EMPTY_PAGE);
+  await page.setContent('<button>Click</button>');
+  await page.click('"Click"');
+  await (context as any)._tracing.stop();
+  await (context as any)._tracing.export(testInfo.outputPath('trace1.zip'));
+
+  await (context as any)._tracing.start({ name: 'test2', screenshots: true, snapshots: true });
+  await page.dblclick('"Click"');
+  await page.close();
+  await (context as any)._tracing.stop();
+  await (context as any)._tracing.export(testInfo.outputPath('trace2.zip'));
+
+  {
+    const { events } = await parseTrace(testInfo.outputPath('trace1.zip'));
+    expect(events[0].type).toBe('context-metadata');
+    expect(events[1].type).toBe('page-created');
+    expect(events.find(e => e.metadata?.apiName === 'page.goto')).toBeTruthy();
+    expect(events.find(e => e.metadata?.apiName === 'page.setContent')).toBeTruthy();
+    expect(events.find(e => e.metadata?.apiName === 'page.click')).toBeTruthy();
+    expect(events.find(e => e.metadata?.apiName === 'page.dblclick')).toBeFalsy();
+    expect(events.find(e => e.metadata?.apiName === 'page.close')).toBeFalsy();
+  }
+
+  {
+    const { events } = await parseTrace(testInfo.outputPath('trace2.zip'));
+    expect(events[0].type).toBe('context-metadata');
+    expect(events[1].type).toBe('page-created');
+    expect(events.find(e => e.metadata?.apiName === 'page.goto')).toBeFalsy();
+    expect(events.find(e => e.metadata?.apiName === 'page.setContent')).toBeFalsy();
+    expect(events.find(e => e.metadata?.apiName === 'page.click')).toBeFalsy();
+    expect(events.find(e => e.metadata?.apiName === 'page.dblclick')).toBeTruthy();
+    expect(events.find(e => e.metadata?.apiName === 'page.close')).toBeTruthy();
+  }
+});
+
+async function parseTrace(file: string): Promise<{ events: any[], resources: Map<string, Buffer> }> {
+  const entries = await new Promise<any[]>(f => {
+    const entries: Promise<any>[] = [];
+    yauzl.open(file, (err, zipFile) => {
+      zipFile.on('entry', entry => {
+        const entryPromise = new Promise(ff => {
+          zipFile.openReadStream(entry, (err, readStream) => {
+            const buffers = [];
+            if (readStream) {
+              readStream.on('data', d => buffers.push(d));
+              readStream.on('end', () => ff({ name: entry.fileName, buffer: Buffer.concat(buffers) }));
+            } else {
+              ff({ name: entry.fileName });
+            }
+          });
+        });
+        entries.push(entryPromise);
+      });
+      zipFile.on('end', () => f(entries));
+    });
+  });
+  const resources = new Map<string, Buffer>();
+  for (const { name, buffer } of await Promise.all(entries))
+    resources.set(name, buffer);
+  const events = resources.get('trace.trace').toString().split('\n').map(line => line ? JSON.parse(line) : false).filter(Boolean);
+  return {
+    events,
+    resources,
+  };
+}