fix(crates): Detect stale upload caches and retry (#98)

The publishing errors also happen locally. When publishing a crate with
`--no-verify`, it seems that the crate caches can go out of date and
`cargo` will fail to publish. We can detect this and retry publishing
with exponential backoff.

It is unclear what the root cause of this is. When cargo updates its
index, it runs a "fast path" request against the GitHub API which seems
to get cached. Waiting for 20-30 seconds seems sufficient to invalidate
those caches.

Author: jan-auer

src/targets/crates.ts

@@ -11,7 +11,11 @@ import { GithubGlobalConfig, TargetConfig } from '../schemas/project_config';
 import { forEachChained } from '../utils/async';
 import { ConfigurationError } from '../utils/errors';
 import { withTempDir } from '../utils/files';
-import { checkExecutableIsPresent, spawnProcess } from '../utils/system';
+import {
+  checkExecutableIsPresent,
+  sleepAsync,
+  spawnProcess,
+} from '../utils/system';
 import { BaseTarget } from './base';
 import { BaseArtifactProvider } from '../artifact_providers/base';
 
@@ -24,6 +28,30 @@ const DEFAULT_CARGO_BIN = 'cargo';
  */
 const CARGO_BIN = process.env.CARGO_BIN || DEFAULT_CARGO_BIN;
 
+/**
+ * A message fragment emitted by cargo when publishing fails due to a missing
+ * dependency. This sometimes indicates a false positive if the cache has not
+ * been updated.
+ */
+const VERSION_ERROR = 'failed to select a version for the requirement';
+
+/**
+ * Maximum number of attempts including the initial one when publishing fails
+ * due to a stale cache. After this number of retries, publishing fails.
+ */
+const MAX_ATTEMPTS = 5;
+
+/**
+ * Initial delay to wait between publish retries in seconds. Exponential backoff
+ * is applied to this delay on retries.
+ */
+const RETRY_DELAY_SECS = 2;
+
+/**
+ * Exponential backoff that is applied to the initial retry delay.
+ */
+const RETRY_EXP_FACTOR = 2;
+
 /** Options for "crates" target */
 export interface CratesTargetOptions extends TargetConfig {
   /** Crates API token */
@@ -203,9 +231,9 @@ export class CratesTarget extends BaseTarget {
 
     const crates = this.getPublishOrder(unorderedCrates);
     logger.debug(
-      `Publishing packages in the following order: [${crates
+      `Publishing packages in the following order: ${crates
         .map(c => c.name)
-        .toString()}]`
+        .join(', ')}`
     );
     return forEachChained(crates, async crate => this.publishPackage(crate));
   }
@@ -227,9 +255,26 @@ export class CratesTarget extends BaseTarget {
       crate.manifest_path
     );
 
-    return spawnProcess(CARGO_BIN, args, {
-      env: { ...process.env, CARGO_REGISTRY_TOKEN: this.cratesConfig.apiToken },
-    });
+    const env = {
+      ...process.env,
+      CARGO_REGISTRY_TOKEN: this.cratesConfig.apiToken,
+    };
+
+    let delay = RETRY_DELAY_SECS;
+    logger.info(`Publishing ${crate.name}`);
+    for (let i = 0; i <= MAX_ATTEMPTS; i++) {
+      try {
+        return await spawnProcess(CARGO_BIN, args, { env });
+      } catch (e) {
+        if (i < MAX_ATTEMPTS && e.message.includes(VERSION_ERROR)) {
+          logger.warn(`Publish failed, trying again in ${delay}s...`);
+          await sleepAsync(delay * 1000);
+          delay *= RETRY_EXP_FACTOR;
+        } else {
+          throw e;
+        }
+      }
+    }
   }
 
   /**