Skip to content

Commit 0228ba4

Browse files
pavelfeldmanpavelfeldman
authored
feat(registry): implement download registry (#1979)
1 parent 062a836 commit 0228ba4

File tree

9 files changed

+129
-57
lines changed

9 files changed

+129
-57
lines changed

‎install-from-github.js

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -41,9 +41,7 @@ const rmAsync = util.promisify(require('rimraf'));
4141
if (outdatedFiles.some(Boolean)) {
4242
console.log(`Rebuilding playwright...`);
4343
try {
44-
execSync('npm run build', {
45-
stdio: 'ignore'
46-
});
44+
execSync('npm run build');
4745
} catch (e) {
4846
}
4947
}
@@ -63,21 +61,17 @@ async function listFiles(dirpath) {
6361
}
6462

6563
async function downloadAllBrowsersAndGenerateProtocolTypes() {
66-
const { downloadBrowserWithProgressBar } = require('./lib/install/browserFetcher');
64+
const { installBrowsersWithProgressBar } = require('./lib/install/installer');
6765
const protocolGenerator = require('./utils/protocol-types-generator');
6866
const browserPaths = require('./lib/install/browserPaths');
67+
const browsersPath = browserPaths.browsersPath(__dirname);
6968
const browsers = require('./browsers.json')['browsers'];
69+
await installBrowsersWithProgressBar(__dirname);
7070
for (const browser of browsers) {
71-
if (await downloadBrowserWithProgressBar(__dirname, browser))
72-
await protocolGenerator.generateProtocol(browser.name, browserPaths.executablePath(__dirname, browser)).catch(console.warn);
71+
const browserPath = browserPaths.browserDirectory(browsersPath, browser);
72+
await protocolGenerator.generateProtocol(browser.name, browserPaths.executablePath(browserPath, browser)).catch(console.warn);
7373
}
7474

75-
// Cleanup stale revisions.
76-
const directories = new Set(await readdirAsync(browserPaths.browsersPath(__dirname)));
77-
for (const browser of browsers)
78-
directories.delete(browserPaths.browserDirectory(__dirname, browser));
79-
await Promise.all([...directories].map(directory => rmAsync(directory)));
80-
8175
try {
8276
console.log('Generating types...');
8377
execSync('npm run generate-types');

‎packages/playwright-chromium/install.js

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,6 @@
1414
* limitations under the License.
1515
*/
1616

17-
const { downloadBrowsersWithProgressBar } = require('playwright-core/lib/install/browserFetcher');
17+
const { installBrowsersWithProgressBar } = require('playwright-core/lib/install/installer');
1818

19-
downloadBrowsersWithProgressBar(__dirname, require('./browsers.json')['browsers']);
19+
installBrowsersWithProgressBar(__dirname);

‎packages/playwright-firefox/install.js

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,6 @@
1414
* limitations under the License.
1515
*/
1616

17-
const { downloadBrowsersWithProgressBar } = require('playwright-core/lib/install/browserFetcher');
17+
const { installBrowsersWithProgressBar } = require('playwright-core/lib/install/installer');
1818

19-
downloadBrowsersWithProgressBar(__dirname, require('./browsers.json')['browsers']);
19+
installBrowsersWithProgressBar(__dirname);

‎packages/playwright-webkit/install.js

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,6 @@
1414
* limitations under the License.
1515
*/
1616

17-
const { downloadBrowsersWithProgressBar } = require('playwright-core/lib/install/browserFetcher');
17+
const { installBrowsersWithProgressBar } = require('playwright-core/lib/install/installer');
1818

19-
downloadBrowsersWithProgressBar(__dirname, require('./browsers.json')['browsers']);
19+
installBrowsersWithProgressBar(__dirname);

‎packages/playwright/install.js

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,6 @@
1414
* limitations under the License.
1515
*/
1616

17-
const { downloadBrowsersWithProgressBar } = require('playwright-core/lib/install/browserFetcher');
17+
const { installBrowsersWithProgressBar } = require('playwright-core/lib/install/installer');
1818

19-
downloadBrowsersWithProgressBar(__dirname, require('./browsers.json')['browsers']);
19+
installBrowsersWithProgressBar(__dirname);

‎src/install/browserFetcher.ts

Lines changed: 5 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -75,32 +75,16 @@ function getDownloadUrl(browserName: BrowserName, platform: BrowserPlatform): st
7575
}
7676
}
7777

78-
export type DownloadOptions = {
79-
browser: BrowserDescriptor,
80-
packagePath: string,
81-
serverHost?: string,
82-
};
83-
8478
function revisionURL(browser: BrowserDescriptor, platform = browserPaths.hostPlatform): string {
8579
const serverHost = getFromENV('PLAYWRIGHT_DOWNLOAD_HOST') || DEFAULT_DOWNLOAD_HOSTS[browser.name];
8680
const urlTemplate = getDownloadUrl(browser.name, platform);
8781
assert(urlTemplate, `ERROR: Playwright does not support ${browser.name} on ${platform}`);
8882
return util.format(urlTemplate, serverHost, browser.revision);
8983
}
9084

91-
export async function downloadBrowsersWithProgressBar(packagePath: string, browsers: BrowserDescriptor[]) {
92-
if (getFromENV('PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD')) {
93-
logPolitely('Skipping browsers download because `PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD` env variable is set');
94-
return false;
95-
}
96-
for (const browser of browsers)
97-
await downloadBrowserWithProgressBar(packagePath, browser);
98-
}
99-
100-
export async function downloadBrowserWithProgressBar(packagePath: string, browser: BrowserDescriptor): Promise<boolean> {
85+
export async function downloadBrowserWithProgressBar(browserPath: string, browser: BrowserDescriptor): Promise<boolean> {
10186
const progressBarName = `${browser.name} v${browser.revision}`;
102-
const targetDir = browserPaths.browserDirectory(packagePath, browser);
103-
if (await existsAsync(targetDir)) {
87+
if (await existsAsync(browserPath)) {
10488
// Already downloaded.
10589
return false;
10690
}
@@ -126,16 +110,16 @@ export async function downloadBrowserWithProgressBar(packagePath: string, browse
126110
const zipPath = path.join(os.tmpdir(), `playwright-download-${browser.name}-${browserPaths.hostPlatform}-${browser.revision}.zip`);
127111
try {
128112
await downloadFile(url, zipPath, progress);
129-
await extract(zipPath, {dir: targetDir});
130-
await chmodAsync(browserPaths.executablePath(packagePath, browser), 0o755);
113+
await extract(zipPath, { dir: browserPath});
114+
await chmodAsync(browserPaths.executablePath(browserPath, browser)!, 0o755);
131115
} catch (e) {
132116
process.exitCode = 1;
133117
throw e;
134118
} finally {
135119
if (await existsAsync(zipPath))
136120
await unlinkAsync(zipPath);
137121
}
138-
logPolitely(`${progressBarName} downloaded to ${targetDir}`);
122+
logPolitely(`${progressBarName} downloaded to ${browserPath}`);
139123
return true;
140124
}
141125

‎src/install/browserPaths.ts

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
import { execSync } from 'child_process';
1919
import * as os from 'os';
2020
import * as path from 'path';
21-
import { assert, getFromENV } from '../helper';
21+
import { getFromENV } from '../helper';
2222

2323
export type BrowserName = 'chromium'|'webkit'|'firefox';
2424
export type BrowserPlatform = 'win32'|'win64'|'mac10.13'|'mac10.14'|'mac10.15'|'linux';
@@ -40,9 +40,10 @@ export const hostPlatform = ((): BrowserPlatform => {
4040
return platform as BrowserPlatform;
4141
})();
4242

43-
function getRelativeExecutablePath(browserName: BrowserName): string[] | undefined {
44-
if (browserName === 'chromium') {
45-
return new Map<BrowserPlatform, string[]>([
43+
export function executablePath(browserPath: string, browser: BrowserDescriptor): string | undefined {
44+
let tokens: string[] | undefined;
45+
if (browser.name === 'chromium') {
46+
tokens = new Map<BrowserPlatform, string[]>([
4647
['linux', ['chrome-linux', 'chrome']],
4748
['mac10.13', ['chrome-mac', 'Chromium.app', 'Contents', 'MacOS', 'Chromium']],
4849
['mac10.14', ['chrome-mac', 'Chromium.app', 'Contents', 'MacOS', 'Chromium']],
@@ -52,8 +53,8 @@ function getRelativeExecutablePath(browserName: BrowserName): string[] | undefin
5253
]).get(hostPlatform);
5354
}
5455

55-
if (browserName === 'firefox') {
56-
return new Map<BrowserPlatform, string[]>([
56+
if (browser.name === 'firefox') {
57+
tokens = new Map<BrowserPlatform, string[]>([
5758
['linux', ['firefox', 'firefox']],
5859
['mac10.13', ['firefox', 'Nightly.app', 'Contents', 'MacOS', 'firefox']],
5960
['mac10.14', ['firefox', 'Nightly.app', 'Contents', 'MacOS', 'firefox']],
@@ -63,8 +64,8 @@ function getRelativeExecutablePath(browserName: BrowserName): string[] | undefin
6364
]).get(hostPlatform);
6465
}
6566

66-
if (browserName === 'webkit') {
67-
return new Map<BrowserPlatform, string[] | undefined>([
67+
if (browser.name === 'webkit') {
68+
tokens = new Map<BrowserPlatform, string[] | undefined>([
6869
['linux', ['pw_run.sh']],
6970
['mac10.13', undefined],
7071
['mac10.14', ['pw_run.sh']],
@@ -73,19 +74,19 @@ function getRelativeExecutablePath(browserName: BrowserName): string[] | undefin
7374
['win64', ['Playwright.exe']],
7475
]).get(hostPlatform);
7576
}
77+
return tokens ? path.join(browserPath, ...tokens) : undefined;
7678
}
7779

7880
export function browsersPath(packagePath: string): string {
7981
const result = getFromENV('PLAYWRIGHT_BROWSERS_PATH');
8082
return result || path.join(packagePath, '.local-browsers');
8183
}
8284

83-
export function browserDirectory(packagePath: string, browser: BrowserDescriptor): string {
84-
return path.join(browsersPath(packagePath), `${browser.name}-${browser.revision}`);
85+
export function browserDirectory(browsersPath: string, browser: BrowserDescriptor): string {
86+
return path.join(browsersPath, `${browser.name}-${browser.revision}`);
8587
}
8688

87-
export function executablePath(packagePath: string, browser: BrowserDescriptor): string {
88-
const relativePath = getRelativeExecutablePath(browser.name);
89-
assert(relativePath, `Unsupported platform for ${browser.name}: ${hostPlatform}`);
90-
return path.join(browserDirectory(packagePath, browser), ...relativePath);
89+
export function isBrowserDirectory(browserPath: string): boolean {
90+
const baseName = path.basename(browserPath);
91+
return baseName.startsWith('chromium-') || baseName.startsWith('firefox-') || baseName.startsWith('webkit-');
9192
}

‎src/install/installer.ts

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
/**
2+
* Copyright Microsoft Corporation. All rights reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
import * as crypto from 'crypto';
18+
import { getFromENV, logPolitely } from '../helper';
19+
import * as fs from 'fs';
20+
import * as path from 'path';
21+
import * as util from 'util';
22+
import * as removeFolder from 'rimraf';
23+
import * as browserPaths from '../install/browserPaths';
24+
import * as browserFetcher from '../install/browserFetcher';
25+
26+
const fsMkdirAsync = util.promisify(fs.mkdir.bind(fs));
27+
const fsExistsAsync = (path: string) => new Promise(f => fs.exists(path, f));
28+
const fsReaddirAsync = util.promisify(fs.readdir.bind(fs));
29+
const fsReadFileAsync = util.promisify(fs.readFile.bind(fs));
30+
const fsUnlinkAsync = util.promisify(fs.unlink.bind(fs));
31+
const fsWriteFileAsync = util.promisify(fs.writeFile.bind(fs));
32+
const removeFolderAsync = util.promisify(removeFolder);
33+
34+
export async function installBrowsersWithProgressBar(packagePath: string) {
35+
const browsersPath = browserPaths.browsersPath(packagePath);
36+
const linksDir = path.join(browsersPath, '.links');
37+
38+
if (getFromENV('PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD')) {
39+
logPolitely('Skipping browsers download because `PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD` env variable is set');
40+
return false;
41+
}
42+
if (!await fsExistsAsync(browsersPath))
43+
await fsMkdirAsync(browsersPath);
44+
if (!await fsExistsAsync(linksDir))
45+
await fsMkdirAsync(linksDir);
46+
47+
await fsWriteFileAsync(path.join(linksDir, sha1(packagePath)), packagePath);
48+
await validateCache(browsersPath, linksDir);
49+
}
50+
51+
async function validateCache(browsersPath: string, linksDir: string) {
52+
// 1. Collect unused downloads and package descriptors.
53+
const allBrowsers: browserPaths.BrowserDescriptor[] = [];
54+
for (const fileName of await fsReaddirAsync(linksDir)) {
55+
const linkPath = path.join(linksDir, fileName);
56+
try {
57+
const linkTarget = (await fsReadFileAsync(linkPath)).toString();
58+
const browsers = JSON.parse((await fsReadFileAsync(path.join(linkTarget, 'browsers.json'))).toString())['browsers'];
59+
allBrowsers.push(...browsers);
60+
} catch (e) {
61+
logPolitely('Failed to process descriptor at ' + fileName);
62+
await fsUnlinkAsync(linkPath).catch(e => {});
63+
}
64+
}
65+
66+
// 2. Delete all unused browsers.
67+
let downloadedBrowsers = (await fsReaddirAsync(browsersPath)).map(file => path.join(browsersPath, file));
68+
downloadedBrowsers = downloadedBrowsers.filter(file => browserPaths.isBrowserDirectory(file));
69+
const directories = new Set<string>(downloadedBrowsers);
70+
directories.delete(path.join(browsersPath, '.links'));
71+
for (const browser of allBrowsers)
72+
directories.delete(browserPaths.browserDirectory(browsersPath, browser));
73+
for (const directory of directories) {
74+
logPolitely('Removing unused browser at ' + directory);
75+
await removeFolderAsync(directory).catch(e => {});
76+
}
77+
78+
// 3. Install missing browsers.
79+
for (const browser of allBrowsers) {
80+
const browserPath = browserPaths.browserDirectory(browsersPath, browser);
81+
await browserFetcher.downloadBrowserWithProgressBar(browserPath, browser);
82+
}
83+
}
84+
85+
function sha1(data: string): string {
86+
const sum = crypto.createHash('sha1');
87+
sum.update(data);
88+
return sum.digest('hex');
89+
}

‎src/server/browserType.ts

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,14 +54,18 @@ export interface BrowserType<Browser> {
5454

5555
export abstract class AbstractBrowserType<Browser> implements BrowserType<Browser> {
5656
private _name: string;
57-
private _executablePath: string;
57+
private _executablePath: string | undefined;
5858

5959
constructor(packagePath: string, browser: browserPaths.BrowserDescriptor) {
6060
this._name = browser.name;
61-
this._executablePath = browserPaths.executablePath(packagePath, browser);
61+
const browsersPath = browserPaths.browsersPath(packagePath);
62+
const browserPath = browserPaths.browserDirectory(browsersPath, browser);
63+
this._executablePath = browserPaths.executablePath(browserPath, browser);
6264
}
6365

6466
executablePath(): string {
67+
if (!this._executablePath)
68+
throw new Error('Browser is not supported on current platform');
6569
return this._executablePath;
6670
}
6771

0 commit comments

Comments
 (0)