feat(downloads): support downloads on cr and wk (#1632)

Author: pavelfeldman

docs/api.md

@@ -14,6 +14,7 @@
 - [class: JSHandle](#class-jshandle)
 - [class: ConsoleMessage](#class-consolemessage)
 - [class: Dialog](#class-dialog)
+- [class: Download](#class-download)
 - [class: Keyboard](#class-keyboard)
 - [class: Mouse](#class-mouse)
 - [class: Request](#class-request)
@@ -191,6 +192,7 @@ Indicates that the browser is connected.
 
 #### browser.newContext([options])
 - `options` <[Object]>
+  - `acceptDownloads` <[boolean]> Whether to automatically download all the attachments. Defaults to `false` where all the downloads are canceled.
   - `ignoreHTTPSErrors` <[boolean]> Whether to ignore HTTPS errors during navigation. Defaults to `false`.
   - `bypassCSP` <[boolean]> Toggles bypassing page's Content-Security-Policy.
   - `viewport` <?[Object]> Sets a consistent viewport for each page. Defaults to an 1280x720 viewport. `null` disables the default viewport.
@@ -230,6 +232,7 @@ Creates a new browser context. It won't share cookies/cache with other browser c
 
 #### browser.newPage([options])
 - `options` <[Object]>
+  - `acceptDownloads` <[boolean]> Whether to automatically download all the attachments. Defaults to `false` where all the downloads are canceled.
   - `ignoreHTTPSErrors` <[boolean]> Whether to ignore HTTPS errors during navigation. Defaults to `false`.
   - `bypassCSP` <[boolean]> Toggles bypassing page's Content-Security-Policy.
   - `viewport` <?[Object]> Sets a consistent viewport for each page. Defaults to an 1280x720 viewport. `null` disables the default viewport.
@@ -629,6 +632,7 @@ page.removeListener('request', logRequest);
 - [event: 'console'](#event-console)
 - [event: 'dialog'](#event-dialog)
 - [event: 'domcontentloaded'](#event-domcontentloaded)
+- [event: 'download'](#event-download)
 - [event: 'filechooser'](#event-filechooser)
 - [event: 'frameattached'](#event-frameattached)
 - [event: 'framedetached'](#event-framedetached)
@@ -729,6 +733,11 @@ Emitted when a JavaScript dialog appears, such as `alert`, `prompt`, `confirm` o
 
 Emitted when the JavaScript [`DOMContentLoaded`](https://developer.mozilla.org/en-US/docs/Web/Events/DOMContentLoaded) event is dispatched.
 
+#### event: 'download'
+- <[Download]>
+
+Emitted when attachment is downloaded. User can access basic file operations on downloaded content via the passed [Download] instance. Browser context must be created with the `acceptDownloads` set to `true` when user needs access to the downloaded content. If `acceptDownloads` is not set or set to `false`, download events are emitted, but the actual download is not performed and user has no access to the downloaded files.
+
 #### event: 'filechooser'
 - <[Object]>
   - `element` <[ElementHandle]> handle to the input element that was clicked
@@ -2971,6 +2980,58 @@ const { chromium } = require('playwright');  // Or 'firefox' or 'webkit'.
 - returns: <[string]> Dialog's type, can be one of `alert`, `beforeunload`, `confirm` or `prompt`.
 
 
+### class: Download
+
+[Download] objects are dispatched by page via the ['download'](#event-download) event.
+
+Note that browser context must be created with the `acceptDownloads` set to `true` when user needs access to the downloaded content. If `acceptDownloads` is not set or set to `false`, download events are emitted, but the actual download is not performed and user has no access to the downloaded files.
+
+All the downloaded files belonging to the browser context are deleted when the browser context is closed. All downloaded files are deleted when the browser closes.
+
+An example of using `Download` class:
+```js
+const [ download ] = await Promise.all([
+  page.waitForEvent('download'),
+  page.click('a')
+]);
+const path = await download.path();
+...
+```
+
+<!-- GEN:toc -->
+- [download.createReadStream()](#downloadcreatereadstream)
+- [download.delete()](#downloaddelete)
+- [download.failure()](#downloadfailure)
+- [download.path()](#downloadpath)
+- [download.url()](#downloadurl)
+<!-- GEN:stop -->
+
+#### download.createReadStream()
+- returns: <[Promise]<null|[Readable]>>
+
+Returns readable stream for current download or `null` if download failed.
+
+#### download.delete()
+- returns: <[Promise]>
+
+Deletes the downloaded file.
+
+#### download.failure()
+- returns: <[Promise]<null|[string]>>
+
+Returns download error if any.
+
+#### download.path()
+- returns: <[Promise]<null|[string]>>
+
+Returns path to the downloaded file in case of successful download.
+
+#### download.url()
+- returns: <[string]>
+
+Returns downloaded url.
+
+
 ### class: Keyboard
 
 Keyboard provides an api for managing a virtual keyboard. The high level api is [`keyboard.type`](#keyboardtypetext-options), which takes raw characters and generates proper keydown, keypress/input, and keyup events on your page.
@@ -4112,6 +4173,6 @@ const { chromium } = require('playwright');
 [number]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Data_structures#Number_type "Number"
 [origin]: https://developer.mozilla.org/en-US/docs/Glossary/Origin "Origin"
 [selector]: https://developer.mozilla.org/en-US/docs/Web/CSS/CSS_Selectors "selector"
-[stream.Readable]: https://nodejs.org/api/stream.html#stream_class_stream_readable "stream.Readable"
+[Readable]: https://nodejs.org/api/stream.html#stream_class_stream_readable "Readable"
 [string]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Data_structures#String_type "String"
 [xpath]: https://developer.mozilla.org/en-US/docs/Web/XPath "xpath"

package.json

@@ -10,7 +10,7 @@
   "playwright": {
     "chromium_revision": "754895",
     "firefox_revision": "1069",
-    "webkit_revision": "1185"
+    "webkit_revision": "1186"
   },
   "scripts": {
     "ctest": "cross-env BROWSER=chromium node test/test.js",

src/api.ts

@@ -19,6 +19,7 @@ export { Browser } from './browser';
 export { BrowserContext } from './browserContext';
 export { ConsoleMessage } from './console';
 export { Dialog } from './dialog';
+export { Download } from './download';
 export { ElementHandle } from './dom';
 export { TimeoutError } from './errors';
 export { Frame } from './frames';

src/browser.ts

@@ -17,6 +17,8 @@
 import { BrowserContext, BrowserContextOptions } from './browserContext';
 import { Page } from './page';
 import { EventEmitter } from 'events';
+import { Download } from './download';
+import { debugProtocol } from './transport';
 
 export interface Browser extends EventEmitter {
   newContext(options?: BrowserContextOptions): Promise<BrowserContext>;
@@ -25,14 +27,38 @@ export interface Browser extends EventEmitter {
   isConnected(): boolean;
   close(): Promise<void>;
   _disconnect(): Promise<void>;
-  _setDebugFunction(debugFunction: (message: string) => void): void;
 }
 
-export async function createPageInNewContext(browser: Browser, options?: BrowserContextOptions): Promise<Page> {
-  const context = await browser.newContext(options);
-  const page = await context.newPage();
-  page._ownedContext = context;
-  return page;
+export abstract class BrowserBase extends EventEmitter implements Browser {
+  _downloadsPath: string = '';
+  private _downloads = new Map<string, Download>();
+  _debugProtocol = debugProtocol;
+
+  abstract newContext(options?: BrowserContextOptions): Promise<BrowserContext>;
+  abstract contexts(): BrowserContext[];
+  abstract isConnected(): boolean;
+  abstract close(): Promise<void>;
+  abstract _disconnect(): Promise<void>;
+
+  async newPage(options?: BrowserContextOptions): Promise<Page> {
+    const context = await this.newContext(options);
+    const page = await context.newPage();
+    page._ownedContext = context;
+    return page;
+  }
+
+  _downloadCreated(page: Page, uuid: string, url: string) {
+    const download = new Download(page, this._downloadsPath, uuid, url);
+    this._downloads.set(uuid, download);
+  }
+
+  _downloadFinished(uuid: string, error: string) {
+    const download = this._downloads.get(uuid);
+    if (!download)
+      return;
+    download._reportFinished(error);
+    this._downloads.delete(uuid);
+  }
 }
 
 export type LaunchType = 'local' | 'server' | 'persistent';

src/browserContext.ts

@@ -22,6 +22,7 @@ import { TimeoutSettings } from './timeoutSettings';
 import * as types from './types';
 import { Events } from './events';
 import { ExtendedEventEmitter } from './extendedEventEmitter';
+import { Download } from './download';
 
 export type BrowserContextOptions = {
   viewport?: types.Size | null,
@@ -38,7 +39,8 @@ export type BrowserContextOptions = {
   httpCredentials?: types.Credentials,
   deviceScaleFactor?: number,
   isMobile?: boolean,
-  hasTouch?: boolean
+  hasTouch?: boolean,
+  acceptDownloads?: boolean
 };
 
 export interface BrowserContext {
@@ -71,6 +73,7 @@ export abstract class BrowserContextBase extends ExtendedEventEmitter implements
   private readonly _closePromise: Promise<Error>;
   private _closePromiseFulfill: ((error: Error) => void) | undefined;
   readonly _permissions = new Map<string, string[]>();
+  readonly _downloads = new Set<Download>();
 
   constructor(options: BrowserContextOptions) {
     super();
@@ -89,13 +92,16 @@ export abstract class BrowserContextBase extends ExtendedEventEmitter implements
   _browserClosed() {
     for (const page of this.pages())
       page._didClose();
-    this._didCloseInternal();
+    this._didCloseInternal(true);
   }
 
-  _didCloseInternal() {
+  async _didCloseInternal(omitDeleteDownloads = false) {
     this._closed = true;
     this.emit(Events.BrowserContext.Close);
     this._closePromiseFulfill!(new Error('Context closed'));
+    if (!omitDeleteDownloads)
+      await Promise.all([...this._downloads].map(d => d.delete()));
+    this._downloads.clear();
   }
 
   // BrowserContext methods.

src/chromium/crBrowser.ts

@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-import { Browser, createPageInNewContext } from '../browser';
+import { BrowserBase } from '../browser';
 import { assertBrowserContextIsNotOwned, BrowserContext, BrowserContextBase, BrowserContextOptions, validateBrowserContextOptions, verifyGeolocation } from '../browserContext';
 import { Events as CommonEvents } from '../events';
 import { assert, debugError, helper } from '../helper';
@@ -29,10 +29,9 @@ import { readProtocolStream } from './crProtocolHelper';
 import { Events } from './events';
 import { Protocol } from './protocol';
 import { CRExecutionContext } from './crExecutionContext';
-import { EventEmitter } from 'events';
 import type { BrowserServer } from '../server/browserServer';
 
-export class CRBrowser extends EventEmitter implements Browser {
+export class CRBrowser extends BrowserBase {
   readonly _connection: CRConnection;
   _session: CRSession;
   private _clientRootSessionPromise: Promise<CRSession> | null = null;
@@ -104,10 +103,6 @@ export class CRBrowser extends EventEmitter implements Browser {
     return Array.from(this._contexts.values());
   }
 
-  async newPage(options?: BrowserContextOptions): Promise<Page> {
-    return createPageInNewContext(this, options);
-  }
-
   _onAttachedToTarget({targetInfo, sessionId, waitingForDebugger}: Protocol.Target.attachedToTargetPayload) {
     const session = this._connection.session(sessionId)!;
     const context = (targetInfo.browserContextId && this._contexts.has(targetInfo.browserContextId)) ?
@@ -250,10 +245,6 @@ export class CRBrowser extends EventEmitter implements Browser {
       this._clientRootSessionPromise = this._connection.createBrowserSession();
     return this._clientRootSessionPromise;
   }
-
-  _setDebugFunction(debugFunction: debug.IDebugger) {
-    this._connection._debugProtocol = debugFunction;
-  }
 }
 
 class CRServiceWorker extends Worker {
@@ -284,12 +275,20 @@ export class CRBrowserContext extends BrowserContextBase {
   }
 
   async _initialize() {
+    const promises: Promise<any>[] = [
+      this._browser._session.send('Browser.setDownloadBehavior', {
+        behavior: this._options.acceptDownloads ? 'allowAndName' : 'deny',
+        browserContextId: this._browserContextId || undefined,
+        downloadPath: this._browser._downloadsPath
+      })
+    ];
     if (this._options.permissions)
-      await this.grantPermissions(this._options.permissions);
+      promises.push(this.grantPermissions(this._options.permissions));
     if (this._options.offline)
-      await this.setOffline(this._options.offline);
+      promises.push(this.setOffline(this._options.offline));
     if (this._options.httpCredentials)
-      await this.setHTTPCredentials(this._options.httpCredentials);
+      promises.push(this.setHTTPCredentials(this._options.httpCredentials));
+    await Promise.all(promises);
   }
 
   pages(): Page[] {
@@ -435,7 +434,7 @@ export class CRBrowserContext extends BrowserContextBase {
     }
     await this._browser._session.send('Target.disposeBrowserContext', { browserContextId: this._browserContextId });
     this._browser._contexts.delete(this._browserContextId);
-    this._didCloseInternal();
+    await this._didCloseInternal();
   }
 
   backgroundPages(): Page[] {

src/chromium/crConnection.ts

@@ -16,8 +16,7 @@
  */
 
 import { assert } from '../helper';
-import * as debug from 'debug';
-import { ConnectionTransport, ProtocolRequest, ProtocolResponse } from '../transport';
+import { ConnectionTransport, ProtocolRequest, ProtocolResponse, debugProtocol } from '../transport';
 import { Protocol } from './protocol';
 import { EventEmitter } from 'events';
 
@@ -35,7 +34,6 @@ export class CRConnection extends EventEmitter {
   private readonly _sessions = new Map<string, CRSession>();
   readonly rootSession: CRSession;
   _closed = false;
-  _debugProtocol: debug.IDebugger;
 
   constructor(transport: ConnectionTransport) {
     super();
@@ -44,8 +42,6 @@ export class CRConnection extends EventEmitter {
     this._transport.onclose = this._onClose.bind(this);
     this.rootSession = new CRSession(this, '', 'browser', '');
     this._sessions.set('', this.rootSession);
-    this._debugProtocol = debug('pw:protocol');
-    (this._debugProtocol as any).color = '34';
   }
 
   static fromSession(session: CRSession): CRConnection {
@@ -61,15 +57,15 @@ export class CRConnection extends EventEmitter {
     const message: ProtocolRequest = { id, method, params };
     if (sessionId)
       message.sessionId = sessionId;
-    if (this._debugProtocol.enabled)
-      this._debugProtocol('SEND ► ' + rewriteInjectedScriptEvaluationLog(message));
+    if (debugProtocol.enabled)
+      debugProtocol('SEND ► ' + rewriteInjectedScriptEvaluationLog(message));
     this._transport.send(message);
     return id;
   }
 
   async _onMessage(message: ProtocolResponse) {
-    if (this._debugProtocol.enabled)
-      this._debugProtocol('◀ RECV ' + JSON.stringify(message));
+    if (debugProtocol.enabled)
+      debugProtocol('◀ RECV ' + JSON.stringify(message));
     if (message.id === kBrowserCloseMessageId)
       return;
     if (message.method === 'Target.attachedToTarget') {

src/chromium/crPage.ts

@@ -94,6 +94,8 @@ export class CRPage implements PageDelegate {
           helper.addEventListener(this._client, 'Page.frameStoppedLoading', event => this._onFrameStoppedLoading(event.frameId)),
           helper.addEventListener(this._client, 'Page.javascriptDialogOpening', event => this._onDialog(event)),
           helper.addEventListener(this._client, 'Page.navigatedWithinDocument', event => this._onFrameNavigatedWithinDocument(event.frameId, event.url)),
+          helper.addEventListener(this._client, 'Page.downloadWillBegin', event => this._onDownloadWillBegin(event)),
+          helper.addEventListener(this._client, 'Page.downloadProgress', event => this._onDownloadProgress(event)),
           helper.addEventListener(this._client, 'Runtime.bindingCalled', event => this._onBindingCalled(event)),
           helper.addEventListener(this._client, 'Runtime.consoleAPICalled', event => this._onConsoleAPI(event)),
           helper.addEventListener(this._client, 'Runtime.exceptionThrown', exception => this._handleException(exception.exceptionDetails)),
@@ -168,7 +170,6 @@ export class CRPage implements PageDelegate {
     promises.push(this._firstNonInitialNavigationCommittedPromise);
     await Promise.all(promises);
   }
-
   didClose() {
     helper.removeEventListeners(this._eventListeners);
     this._networkManager.dispose();
@@ -356,6 +357,17 @@ export class CRPage implements PageDelegate {
     this._page._onFileChooserOpened(handle);
   }
 
+  _onDownloadWillBegin(payload: Protocol.Page.downloadWillBeginPayload) {
+    this._browserContext._browser._downloadCreated(this._page, payload.guid, payload.url);
+  }
+
+  _onDownloadProgress(payload: Protocol.Page.downloadProgressPayload) {
+    if (payload.state === 'completed')
+      this._browserContext._browser._downloadFinished(payload.guid, '');
+    if (payload.state === 'canceled')
+      this._browserContext._browser._downloadFinished(payload.guid, 'canceled');
+  }
+
   async updateExtraHTTPHeaders(): Promise<void> {
     const headers = network.mergeHeaders([
       this._browserContext._options.extraHTTPHeaders,