Merge pull request #99 from amtrack/feat/read-in-parallel

feat: read metadata of the same type in parallel in chunks of 10

Author: amtrack

src/commands/force/source/read.ts

@@ -1,14 +1,22 @@
 import type { MetadataType } from "@jsforce/jsforce-node/lib/api/metadata.js";
-import { Flags, SfCommand, requiredOrgFlagWithDeprecations } from "@salesforce/sf-plugins-core";
-import { ComponentSetBuilder, SourceComponent } from "@salesforce/source-deploy-retrieve";
+import {
+  Flags,
+  SfCommand,
+  requiredOrgFlagWithDeprecations,
+} from "@salesforce/sf-plugins-core";
+import {
+  ComponentSetBuilder,
+  SourceComponent,
+} from "@salesforce/source-deploy-retrieve";
 import { filePathsFromMetadataComponent } from "@salesforce/source-deploy-retrieve/lib/src/utils/filePathGenerator.js";
 import { mkdir, writeFile } from "fs/promises";
 import { dirname, join } from "path";
 import { convertToXml, parseCommaSeparatedValues } from "../../../utils.js";
 
 export class SourceReadCommand extends SfCommand<any> {
   public static readonly summary = "Read Metadata using the CRUD Metadata API";
-  public static readonly description = "Read Metadata using the CRUD Metadata API";
+  public static readonly description =
+    "Read Metadata using the CRUD Metadata API";
 
   public static readonly examples = [
     `$ <%= config.bin %> <%= command.id %> -m "Profile:Admin"`,
@@ -39,29 +47,67 @@ export class SourceReadCommand extends SfCommand<any> {
     const defaultPackageDirectory = this.project.getDefaultPackage().path;
     const sourcePaths = packageDirectories.map((dir) => dir.path);
     const componentSet = await ComponentSetBuilder.build({
-      sourcepath: flags.sourcepath && parseCommaSeparatedValues(flags.sourcepath),
+      sourcepath:
+        flags.sourcepath && parseCommaSeparatedValues(flags.sourcepath),
       ...(flags.metadata && {
         metadata: {
           metadataEntries: parseCommaSeparatedValues(flags.metadata),
           directoryPaths: sourcePaths,
-        }
+        },
       }),
     });
-    for (const component of componentSet) {
-      this.log("reading", `${component.type.name}:${component.fullName}`, "...");
-      const mdJson = await conn.metadata.read(component.type.name as MetadataType, component.fullName);
-      let filePath;
-      if (component instanceof SourceComponent) {
-        filePath = component.xml;
-      } else {
-        filePath = filePathsFromMetadataComponent(component, join(defaultPackageDirectory, "main", "default")).find(
-          (p) => p.endsWith(`.${component.type.suffix}-meta.xml`),
+    const manifestObject = await componentSet.getObject();
+    const sourceComponents = componentSet.getSourceComponents();
+    for (const typeMember of manifestObject.Package.types) {
+      const typeName = typeMember.name;
+      // https://developer.salesforce.com/docs/atlas.en-us.api_meta.meta/api_meta/meta_readMetadata.htm
+      const chunkSize =
+        typeName === "CustomApplication" || typeName === "CustomMetadata"
+          ? 200
+          : 10;
+      for (const chunkOfMemberNames of chunk(typeMember.members, chunkSize)) {
+        const componentNames = chunkOfMemberNames.map(
+          (memberName) => `${typeName}:${memberName}`
+        );
+        this.log("reading", `${componentNames.join(", ")}`, "...");
+        const metadataResults = await conn.metadata.read(
+          typeName as MetadataType,
+          chunkOfMemberNames
         );
-        await mkdir(dirname(filePath), { recursive: true });
+        for (const metadataResult of metadataResults) {
+          let filePath;
+          const component =
+            sourceComponents.find(
+              (cmp) =>
+                cmp.type.name === typeName &&
+                cmp.fullName === metadataResult.fullName
+            ) ||
+            componentSet.find(
+              (cmp) =>
+                cmp.type.name === typeName &&
+                cmp.fullName === metadataResult.fullName
+            );
+          if (component instanceof SourceComponent) {
+            filePath = component.xml;
+          } else {
+            filePath = filePathsFromMetadataComponent(
+              component,
+              join(defaultPackageDirectory, "main", "default")
+            ).find((p) => p.endsWith(`.${component.type.suffix}-meta.xml`));
+            await mkdir(dirname(filePath), { recursive: true });
+          }
+          await writeFile(filePath, convertToXml(component, metadataResult));
+        }
       }
-      await writeFile(filePath, convertToXml(component, mdJson));
     }
-
     return;
   }
 }
+
+const chunk = (input, size) => {
+  return input.reduce((arr, item, idx) => {
+    return idx % size === 0
+      ? [...arr, [item]]
+      : [...arr.slice(0, -1), [...arr.slice(-1)[0], item]];
+  }, []);
+};