chore: download browser downloads in-memory

Author: mxschmitt

packages/playwright-core/bundles/zip/src/third_party/extract-zip.d.ts

@@ -49,7 +49,7 @@ declare namespace extract {
 }
 
 declare function extract(
-  zipPath: string,
+  zipPath: Buffer,
   opts: extract.Options,
 ): Promise<void>;
 

packages/playwright-core/bundles/zip/src/third_party/extract-zip.js

@@ -33,20 +33,21 @@ const { promisify } = require('util')
 const stream = require('stream')
 const yauzl = require('yauzl')
 
-const openZip = promisify(yauzl.open)
+const openZipFromBuffer = promisify(yauzl.fromBuffer)
 const pipeline = promisify(stream.pipeline)
 
 class Extractor {
-  constructor (zipPath, opts) {
-    this.zipPath = zipPath
+  constructor (zipFile, opts) {
+    /** @type {Buffer} */
+    this.zipFile = zipFile
     this.opts = opts
   }
 
   async extract () {
-    debug('opening', this.zipPath, 'with opts', this.opts)
+    debug('opening', `${this.zipFile.byteLength} bytes`, 'with opts', this.opts);
 
-    this.zipfile = await openZip(this.zipPath, { lazyEntries: true })
-    this.canceled = false
+    this.zipfile = await openZipFromBuffer(this.zipFile, { lazyEntries: true });
+    this.canceled = false;
 
     return new Promise((resolve, reject) => {
       this.zipfile.on('error', err => {
@@ -55,7 +56,7 @@ class Extractor {
       })
       this.zipfile.readEntry()
 
-      this.zipfile.on('close', () => {
+      this.zipfile.on('end', () => {
         if (!this.canceled) {
           debug('zip extraction complete')
           resolve()
@@ -186,7 +187,7 @@ class Extractor {
   }
 }
 
-module.exports = async function (zipPath, opts) {
+module.exports = async function (zipFile, opts) {
   debug('creating target directory', opts.dir)
 
   if (!path.isAbsolute(opts.dir)) {
@@ -195,5 +196,5 @@ module.exports = async function (zipPath, opts) {
 
   await fs.mkdir(opts.dir, { recursive: true })
   opts.dir = await fs.realpath(opts.dir)
-  return new Extractor(zipPath, opts).extract()
+  return new Extractor(zipFile, opts).extract()
 }

packages/playwright-core/src/server/registry/browserFetcher.ts

@@ -17,7 +17,6 @@
 
 import * as childProcess from 'child_process';
 import fs from 'fs';
-import os from 'os';
 import path from 'path';
 
 import { debugLogger } from '../utils/debugLogger';
@@ -37,20 +36,17 @@ export async function downloadBrowserWithProgressBar(title: string, browserDirec
     return false;
   }
 
-  const zipPath = path.join(os.tmpdir(), downloadFileName);
   try {
     const retryCount = 5;
     for (let attempt = 1; attempt <= retryCount; ++attempt) {
       debugLogger.log('install', `downloading ${title} - attempt #${attempt}`);
       const url = downloadURLs[(attempt - 1) % downloadURLs.length];
       logPolitely(`Downloading ${title}` + colors.dim(` from ${url}`));
-      const { error } = await downloadBrowserWithProgressBarOutOfProcess(title, browserDirectory, url, zipPath, executablePath, downloadSocketTimeout);
+      const { error } = await downloadBrowserWithProgressBarOutOfProcess(title, browserDirectory, url, executablePath, downloadSocketTimeout);
       if (!error) {
         debugLogger.log('install', `SUCCESS installing ${title}`);
         break;
       }
-      if (await existsAsync(zipPath))
-        await fs.promises.unlink(zipPath);
       if (await existsAsync(browserDirectory))
         await fs.promises.rmdir(browserDirectory, { recursive: true });
       const errorMessage = error?.message || '';
@@ -62,9 +58,6 @@ export async function downloadBrowserWithProgressBar(title: string, browserDirec
     debugLogger.log('install', `FAILED installation ${title} with error: ${e}`);
     process.exitCode = 1;
     throw e;
-  } finally {
-    if (await existsAsync(zipPath))
-      await fs.promises.unlink(zipPath);
   }
   logPolitely(`${title} downloaded to ${browserDirectory}`);
   return true;
@@ -75,7 +68,7 @@ export async function downloadBrowserWithProgressBar(title: string, browserDirec
  * Thats why we execute it in a separate process and check manually if the destination file exists.
  * https://github.com/microsoft/playwright/issues/17394
  */
-function downloadBrowserWithProgressBarOutOfProcess(title: string, browserDirectory: string, url: string, zipPath: string, executablePath: string | undefined, socketTimeout: number): Promise<{ error: Error | null }> {
+function downloadBrowserWithProgressBarOutOfProcess(title: string, browserDirectory: string, url: string, executablePath: string | undefined, socketTimeout: number): Promise<{ error: Error | null }> {
   const cp = childProcess.fork(path.join(__dirname, 'oopDownloadBrowserMain.js'));
   const promise = new ManualPromise<{ error: Error | null }>();
   const progress = getDownloadProgress();
@@ -101,12 +94,11 @@ function downloadBrowserWithProgressBarOutOfProcess(title: string, browserDirect
 
   debugLogger.log('install', `running download:`);
   debugLogger.log('install', `-- from url: ${url}`);
-  debugLogger.log('install', `-- to location: ${zipPath}`);
+  debugLogger.log('install', `-- to location: ${browserDirectory}`);
   const downloadParams: DownloadParams = {
     title,
     browserDirectory,
     url,
-    zipPath,
     executablePath,
     socketTimeout,
     userAgent: getUserAgent(),

packages/playwright-core/src/server/registry/oopDownloadBrowserMain.ts

@@ -25,7 +25,6 @@ export type DownloadParams = {
   title: string;
   browserDirectory: string;
   url: string;
-  zipPath: string;
   executablePath: string | undefined;
   socketTimeout: number;
   userAgent: string;
@@ -43,11 +42,11 @@ function browserDirectoryToMarkerFilePath(browserDirectory: string): string {
   return path.join(browserDirectory, 'INSTALLATION_COMPLETE');
 }
 
-function downloadFile(options: DownloadParams): Promise<void> {
+function downloadFile(options: DownloadParams): Promise<Buffer> {
   let downloadedBytes = 0;
   let totalBytes = 0;
 
-  const promise = new ManualPromise<void>();
+  const promise = new ManualPromise<Buffer>();
 
   httpRequest({
     url: options.url,
@@ -73,21 +72,22 @@ function downloadFile(options: DownloadParams): Promise<void> {
     }
     totalBytes = parseInt(response.headers['content-length'] || '0', 10);
     log(`-- total bytes: ${totalBytes}`);
-    const file = fs.createWriteStream(options.zipPath);
-    file.on('finish', () => {
+    const chunks: Buffer[] = [];
+    response.on('end', () => {
       if (downloadedBytes !== totalBytes) {
         log(`-- download failed, size mismatch: ${downloadedBytes} != ${totalBytes}`);
         promise.reject(new Error(`Download failed: size mismatch, file size: ${downloadedBytes}, expected size: ${totalBytes} URL: ${options.url}`));
       } else {
         log(`-- download complete, size: ${downloadedBytes}`);
-        promise.resolve();
+        promise.resolve(Buffer.concat(chunks));
       }
     });
-    file.on('error', error => promise.reject(error));
-    response.pipe(file);
-    response.on('data', onData);
+    response.on('data', chunk => {
+      chunks.push(chunk);
+      downloadedBytes += chunk.length;
+      progress(downloadedBytes, totalBytes);
+    });
     response.on('error', (error: any) => {
-      file.close();
       if (error?.code === 'ECONNRESET') {
         log(`-- download failed, server closed connection`);
         promise.reject(new Error(`Download failed: server closed connection. URL: ${options.url}`));
@@ -98,18 +98,13 @@ function downloadFile(options: DownloadParams): Promise<void> {
     });
   }, (error: any) => promise.reject(error));
   return promise;
-
-  function onData(chunk: string) {
-    downloadedBytes += chunk.length;
-    progress(downloadedBytes, totalBytes);
-  }
 }
 
 async function main(options: DownloadParams) {
-  await downloadFile(options);
+  const zipFile = await downloadFile(options);
   log(`SUCCESS downloading ${options.title}`);
   log(`extracting archive`);
-  await extract(options.zipPath, { dir: options.browserDirectory });
+  await extract(zipFile, { dir: options.browserDirectory });
   if (options.executablePath) {
     log(`fixing permissions at ${options.executablePath}`);
     await fs.promises.chmod(options.executablePath, 0o755);

tests/library/browsercontext-har.spec.ts

@@ -283,7 +283,7 @@ it('should round-trip extracted har.zip', async ({ contextFactory, server }, tes
   await context1.close();
 
   const harDir = testInfo.outputPath('hardir');
-  await extractZip(harPath, { dir: harDir });
+  await extractZip(await fs.promises.readFile(harPath), { dir: harDir });
 
   const context2 = await contextFactory();
   await context2.routeFromHAR(path.join(harDir, 'har.har'));

tests/playwright-test/reporter-blob.spec.ts

@@ -1438,7 +1438,7 @@ test('blob report should include version', async ({ runInlineTest }) => {
 });
 
 async function extractReport(reportZipFile: string, unzippedReportDir: string): Promise<any[]> {
-  await extractZip(reportZipFile, { dir: unzippedReportDir });
+  await extractZip(await fs.promises.readFile(reportZipFile), { dir: unzippedReportDir });
   const reportFile = path.join(unzippedReportDir, 'report.jsonl');
   const data = await fs.promises.readFile(reportFile, 'utf8');
   const events = data.split('\n').filter(Boolean).map(line => JSON.parse(line));