generator: Cache the lld binary after extracting it from the LLVM archive (#149)

Extracting `lld` from the `llvm` archive takes about 40s on my machine, and
after the binary has been extracted a 5.5GB directory is left behind in the
`Artifacts` directory.

This change uses `tar` filters to reduce the number of files unpacked from
the archive and caches the resulting `lld` binary.

Excluding the time to download the archives, building an SDK from
Debian packages before `lld` has been cached takes about 57s on an
M3 MacBook Air.   A subsequent build with `lld` cached takes about
17s.

The SDKs generated by the old and new methods are identical.

This PR adds `CacheKey` conformance to `Array`, as suggested by 
@yingguqing in #106.   Adding this conformance to `Range` was not
necessary in this case.

--

* generator: Adopt CacheKey conformance for [FilePath.Component]

This is needed to cache arrays of [FilePath.Component], so a subsequent commit
can cache the slow extraction of the `ld.lld` binary from the `llvm` tar archive.

This problem was also reported by @yingguqing in #106, although Range conformance
does not seem to be required to cache lld.

Suggested-by: @yingguqing

* generator: Cache the lld binary after extracting it from the LLVM archive

Extracting `lld` from the `llvm` archive takes about 40s on my
machine, and after the binary has been extracted a 5.5GB directory
is left in the `Artifacts` directory.

This change uses `tar` filters to reduce the number of files unpacked from
the archive and caches the resulting `lld` binary.

Excluding the time to download the archives, building an SDK from
Debian packages before `lld` has been cached takes about 57s on an
M3 MacBook Air.   A subsequent build with `lld` cached takes about
17s.

The SDKs generated by the old and new methods are identical.

Author: euanh

Sources/Helpers/Vendor/QueryEngine/CacheKey.swift

@@ -175,3 +175,11 @@ extension URL: LeafCacheKey {
     self.description.hash(with: &hashFunction)
   }
 }
+
+extension Array: CacheKey where Element == FilePath.Component {
+  func hash(with hashFunction: inout some HashFunction) {
+    String(reflecting: Self.self).hash(with: &hashFunction)
+    map(\.string).joined(separator: "\n").hash(with: &hashFunction)
+  }
+}
+

Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Unpack.swift

@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 import Helpers
+
 import struct SystemPackage.FilePath
 
 let unusedDarwinPlatforms = [
@@ -99,16 +100,21 @@ extension SwiftSDKGenerator {
       FilePath.Component(llvmArtifact.localPath.stem!)!.stem
     )
     try self.createDirectoryIfNeeded(at: untarDestination)
-    try await self.untar(
-      file: llvmArtifact.localPath,
-      into: untarDestination,
-      stripComponents: 1
-    )
 
     let unpackedLLDPath: FilePath
     if llvmArtifact.isPrebuilt {
-      unpackedLLDPath = untarDestination.appending("bin/lld")
+      unpackedLLDPath = try await engine[TarExtractQuery(
+        file: llvmArtifact.localPath,
+        into: untarDestination,
+        outputBinarySubpath: ["bin", "lld"],
+        stripComponents: 1
+      )].path
     } else {
+      try await self.untar(
+        file: llvmArtifact.localPath,
+        into: untarDestination,
+        stripComponents: 1
+      )
       unpackedLLDPath = try await engine[CMakeBuildQuery(
         sourcesDirectory: untarDestination,
         outputBinarySubpath: ["bin", "lld"],

Sources/SwiftSDKGenerator/Queries/TarExtractQuery.swift

@@ -0,0 +1,36 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift open source project
+//
+// Copyright (c) 2022-202Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import Helpers
+
+import struct SystemPackage.FilePath
+
+struct TarExtractQuery: CachingQuery {
+  var file: FilePath  // Archive to extract
+  var into: FilePath  // Destination for unpacked archive
+  var outputBinarySubpath: [FilePath.Component]
+  var stripComponents: Int? = nil
+
+  func run(engine: QueryEngine) async throws -> FilePath {
+    let stripComponentsOption = stripComponents.map { " --strip-components \($0)" } ?? ""
+    let archivePath = self.file
+    let destinationPath = self.into
+
+    try await Shell.run(
+      #"tar -C "\#(destinationPath)" -x -f "\#(archivePath)" \#(stripComponentsOption) \#(FilePath("*").appending(outputBinarySubpath))"#,
+      logStdout: true
+    )
+
+    // We must return a path to a single file which the cache will track
+    return destinationPath.appending(outputBinarySubpath)
+  }
+}