Run install and pack when fetching git dependencies with a prepare script. (#3553)

Author: Volune

__tests__/fetchers.js

@@ -76,6 +76,32 @@ test('GitFetcher.fetch', async () => {
   expect(name).toBe('beeper');
 });
 
+test('GitFetcher.fetch with prepare script', async () => {
+  const dir = await mkdir('git-fetcher-with-prepare');
+  const fetcher = new GitFetcher(
+    dir,
+    {
+      type: 'git',
+      reference: 'https://github.com/Volune/test-js-git-repo',
+      hash: '96e838bcc908ed424666b4b04efe802fd4c8bccd',
+      registry: 'npm',
+    },
+    (await Config.create()),
+  );
+  await fetcher.fetch();
+  const name = (await fs.readJson(path.join(dir, 'package.json'))).name;
+  expect(name).toBe('test-js-git-repo');
+  const dependencyName = (await fs.readJson(path.join(dir, 'dependency-package.json'))).name;
+  expect(dependencyName).toBe('beeper');
+  // The file "prepare.js" is not in "files" list
+  expect(await fs.exists(path.join(dir, 'prepare.js'))).toBe(false);
+  // Check executed lifecycle scripts
+  expect(await fs.exists(path.join(dir, 'generated', 'preinstall'))).toBe(true);
+  expect(await fs.exists(path.join(dir, 'generated', 'install'))).toBe(true);
+  expect(await fs.exists(path.join(dir, 'generated', 'postinstall'))).toBe(true);
+  expect(await fs.exists(path.join(dir, 'generated', 'prepublish'))).toBe(false);
+});
+
 test('TarballFetcher.fetch', async () => {
   const dir = await mkdir('tarball-fetcher');
   const fetcher = new TarballFetcher(

src/cli/commands/install.js

@@ -823,6 +823,13 @@ export function setFlags(commander: Object) {
   commander.option('-T, --save-tilde', 'DEPRECATED');
 }
 
+export async function install(config: Config, reporter: Reporter, flags: Object, lockfile: Lockfile): Promise<void> {
+  await wrapLifecycle(config, flags, async () => {
+    const install = new Install(flags, config, reporter, lockfile);
+    await install.init();
+  });
+}
+
 export async function run(config: Config, reporter: Reporter, flags: Object, args: Array<string>): Promise<void> {
   let lockfile;
   if (flags.lockfile === false) {
@@ -855,10 +862,7 @@ export async function run(config: Config, reporter: Reporter, flags: Object, arg
     throw new MessageError(reporter.lang('installCommandRenamed', `yarn ${command} ${exampleArgs.join(' ')}`));
   }
 
-  await wrapLifecycle(config, flags, async () => {
-    const install = new Install(flags, config, reporter, lockfile);
-    await install.init();
-  });
+  await install(config, reporter, flags, lockfile);
 }
 
 export async function wrapLifecycle(config: Config, flags: Object, factory: () => Promise<void>): Promise<void> {
@@ -871,7 +875,9 @@ export async function wrapLifecycle(config: Config, flags: Object, factory: () =
   await config.executeLifecycleScript('postinstall');
 
   if (!config.production) {
-    await config.executeLifecycleScript('prepublish');
+    if (!config.disablePrepublish) {
+      await config.executeLifecycleScript('prepublish');
+    }
     await config.executeLifecycleScript('prepare');
   }
 }

src/cli/commands/pack.js

@@ -48,7 +48,10 @@ const NEVER_IGNORE = ignoreLinesToRegex([
   '!/+(changes|changelog|history)*',
 ]);
 
-export async function pack(config: Config, dir: string): Promise<stream$Duplex> {
+export async function packTarball(
+  config: Config,
+  {mapHeader}: {mapHeader?: Object => Object} = {},
+): Promise<stream$Duplex> {
   const pkg = await config.readRootManifest();
   const {bundledDependencies, main, files: onlyFiles} = pkg;
 
@@ -123,10 +126,15 @@ export async function pack(config: Config, dir: string): Promise<stream$Duplex>
       header.name = `package${suffix}`;
       delete header.uid;
       delete header.gid;
-      return header;
+      return mapHeader ? mapHeader(header) : header;
     },
   });
 
+  return packer;
+}
+
+export async function pack(config: Config, dir: string): Promise<stream$Duplex> {
+  const packer = await packTarball(config);
   const compressor = packer.pipe(new zlib.Gzip());
 
   return compressor;

src/config.js

@@ -39,6 +39,7 @@ export type ConfigOptions = {
   ignoreEngines?: boolean,
   cafile?: ?string,
   production?: boolean,
+  disablePrepublish?: boolean,
   binLinks?: boolean,
   networkConcurrency?: number,
   childConcurrency?: number,
@@ -144,6 +145,8 @@ export default class Config {
 
   production: boolean;
 
+  disablePrepublish: boolean;
+
   nonInteractive: boolean;
 
   workspacesEnabled: boolean;
@@ -328,6 +331,8 @@ export default class Config {
     this.ignorePlatform = !!opts.ignorePlatform;
     this.ignoreScripts = !!opts.ignoreScripts;
 
+    this.disablePrepublish = !!opts.disablePrepublish;
+
     this.nonInteractive = !!opts.nonInteractive;
 
     this.requestManager.setOptions({

src/fetchers/git-fetcher.js

@@ -7,6 +7,10 @@ import Git from '../util/git.js';
 import * as fsUtil from '../util/fs.js';
 import * as constants from '../constants.js';
 import * as crypto from '../util/crypto.js';
+import {install} from '../cli/commands/install.js';
+import Lockfile from '../lockfile/wrapper.js';
+import Config from '../config.js';
+import {packTarball} from '../cli/commands/pack.js';
 
 const tarFs = require('tar-fs');
 const url = require('url');
@@ -15,6 +19,8 @@ const fs = require('fs');
 
 const invariant = require('invariant');
 
+const PACKED_FLAG = '1';
+
 export default class GitFetcher extends BaseFetcher {
   async setupMirrorFromCache(): Promise<?string> {
     const tarballMirrorPath = this.getTarballMirrorPath();
@@ -90,11 +96,7 @@ export default class GitFetcher extends BaseFetcher {
     }
 
     return new Promise((resolve, reject) => {
-      const untarStream = tarFs.extract(this.dest, {
-        dmode: 0o555, // all dirs should be readable
-        fmode: 0o444, // all files should be readable
-        chown: false, // don't chown. just leave as it is
-      });
+      const untarStream = this._createUntarStream(this.dest);
 
       const hashStream = new crypto.HashStream();
 
@@ -131,22 +133,130 @@ export default class GitFetcher extends BaseFetcher {
     const gitUrl = Git.npmUrlToGitUrl(this.reference);
     const git = new Git(this.config, gitUrl, hash);
     await git.init();
-    await git.clone(this.dest);
+
+    const manifestFile = await git.getFile('package.json');
+    if (!manifestFile) {
+      throw new MessageError(this.reporter.lang('couldntFindPackagejson', gitUrl));
+    }
+    const scripts = JSON.parse(manifestFile).scripts;
+    const hasPrepareScript = Boolean(scripts && scripts.prepare);
+
+    if (hasPrepareScript) {
+      await this.fetchFromInstallAndPack(git);
+    } else {
+      await this.fetchFromGitArchive(git);
+    }
+
+    return {
+      hash,
+    };
+  }
+
+  async fetchFromInstallAndPack(git: Git): Promise<void> {
+    const prepareDirectory = this.config.getTemp(`${crypto.hash(git.gitUrl.repository)}.${git.hash}.prepare`);
+    await fsUtil.unlink(prepareDirectory);
+
+    await git.clone(prepareDirectory);
+
+    const [prepareConfig, prepareLockFile] = await Promise.all([
+      Config.create(
+        {
+          cwd: prepareDirectory,
+          disablePrepublish: true,
+        },
+        this.reporter,
+      ),
+      Lockfile.fromDirectory(prepareDirectory, this.reporter),
+    ]);
+    await install(prepareConfig, this.reporter, {}, prepareLockFile);
 
     const tarballMirrorPath = this.getTarballMirrorPath();
     const tarballCachePath = this.getTarballCachePath();
 
+    if (tarballMirrorPath) {
+      await this._packToTarball(prepareConfig, tarballMirrorPath);
+    }
+    if (tarballCachePath) {
+      await this._packToTarball(prepareConfig, tarballCachePath);
+    }
+
+    await this._packToDirectory(prepareConfig, this.dest);
+
+    await fsUtil.unlink(prepareDirectory);
+  }
+
+  async _packToTarball(config: Config, path: string): Promise<void> {
+    const tarballStream = await this._createTarballStream(config);
+    await new Promise((resolve, reject) => {
+      const writeStream = fs.createWriteStream(path);
+      tarballStream.on('error', reject);
+      writeStream.on('error', reject);
+      writeStream.on('end', resolve);
+      writeStream.on('open', () => {
+        tarballStream.pipe(writeStream);
+      });
+      writeStream.once('finish', resolve);
+    });
+  }
+
+  async _packToDirectory(config: Config, dest: string): Promise<void> {
+    const tarballStream = await this._createTarballStream(config);
+    await new Promise((resolve, reject) => {
+      const untarStream = this._createUntarStream(dest);
+      tarballStream.on('error', reject);
+      untarStream.on('error', reject);
+      untarStream.on('end', resolve);
+      untarStream.once('finish', resolve);
+      tarballStream.pipe(untarStream);
+    });
+  }
+
+  _createTarballStream(config: Config): Promise<stream$Duplex> {
+    let savedPackedHeader = false;
+    return packTarball(config, {
+      mapHeader(header: Object): Object {
+        if (!savedPackedHeader) {
+          savedPackedHeader = true;
+          header.pax = header.pax || {};
+          // add a custom data on the first header
+          // in order to distinguish a tar from "git archive" and a tar from "pack" command
+          header.pax.packed = PACKED_FLAG;
+        }
+        return header;
+      },
+    });
+  }
+
+  _createUntarStream(dest: string): stream$Writable {
+    const PREFIX = 'package/';
+    let isPackedTarball = undefined;
+    return tarFs.extract(dest, {
+      dmode: 0o555, // all dirs should be readable
+      fmode: 0o444, // all files should be readable
+      chown: false, // don't chown. just leave as it is
+      map: header => {
+        if (isPackedTarball === undefined) {
+          isPackedTarball = header.pax && header.pax.packed === PACKED_FLAG;
+        }
+        if (isPackedTarball) {
+          header.name = header.name.substr(PREFIX.length);
+        }
+      },
+    });
+  }
+
+  async fetchFromGitArchive(git: Git): Promise<void> {
+    await git.clone(this.dest);
+    const tarballMirrorPath = this.getTarballMirrorPath();
+    const tarballCachePath = this.getTarballCachePath();
+
     if (tarballMirrorPath) {
       await git.archive(tarballMirrorPath);
     }
 
     if (tarballCachePath) {
       await git.archive(tarballCachePath);
     }
-
-    return {
-      hash,
-    };
   }
 
   async _fetch(): Promise<FetchedOverride> {